loic@fold:~/software/ceph/teuthology$ cat /tmp/A teuthology_branch: master kernel: kdb: true sha1: 8362a1290d075f376ba68521ffb3b42ecaaecfea nuke-on-error: false machine_type: vps os_type: ubuntu branch: wip-8475 suite: upgrade:firefly-x:stress-split tasks: - chef: - clock.check: overrides: workunit: sha1: 0aed380a61114710cf5edf49472d1c91e31991ac s3tests: branch: master install: ceph: sha1: 0aed380a61114710cf5edf49472d1c91e31991ac ceph: sha1: 0aed380a61114710cf5edf49472d1c91e31991ac conf: mon: debug ms: 1 debug mon: 20 debug paxos: 20 osd: debug ms: 1 debug osd: 20 debug filestore: 20 debug journal: 20 log-whitelist: - slow request ceph-deploy: branch: dev: wip-8475 conf: mon: osd default pool size: 2 debug mon: 1 debug paxos: 20 debug ms: 20 client: log file: /var/log/ceph/ceph-$name.$pid.log admin_socket: branch: wip-8475 loic@fold:~/software/ceph/teuthology$ ./virtualenv/bin/teuthology -v --owner loic@dachary.org targets.yaml /tmp/A /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/0-cluster/start.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/1-firefly-install/firefly.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/2-partial-upgrade/firsthalf.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/3-thrash/default.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/4-mon/mona.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/5-workload/ec-readwrite.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/6-next-mon/monb.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/7-workload/radosbench.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/8-next-mon/monc.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/9-workload/rbd-python.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/9-workload/rgw-s3tests.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/9-workload/snaps-many-objects.yaml /home/loic/software/ceph/ceph-qa-suite/suites/upgrade/firefly-x/stress-split/distros/ubuntu_14.04.yaml 2014-07-01 16:08:32,124.124 WARNING:teuthology.report:No job_id found; not reporting results 2014-07-01 16:08:32,144.144 DEBUG:teuthology.run:Config: branch: wip-8475 kernel: kdb: true sha1: 8362a1290d075f376ba68521ffb3b42ecaaecfea machine_type: vps nuke-on-error: false os_type: ubuntu os_version: '14.04' overrides: admin_socket: branch: wip-8475 ceph: conf: mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug filestore: 20 debug journal: 20 debug ms: 1 debug osd: 20 log-whitelist: - slow request - wrongly marked me down - objects unfound and apparently lost - log bound mismatch sha1: 0aed380a61114710cf5edf49472d1c91e31991ac ceph-deploy: branch: dev: wip-8475 conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: debug mon: 1 debug ms: 20 debug paxos: 20 osd default pool size: 2 install: ceph: sha1: 0aed380a61114710cf5edf49472d1c91e31991ac s3tests: branch: master workunit: sha1: 0aed380a61114710cf5edf49472d1c91e31991ac roles: - - mon.a - mon.b - mds.a - osd.0 - osd.1 - osd.2 - - osd.3 - osd.4 - osd.5 - mon.c - - client.0 suite: upgrade:firefly-x:stress-split targets: ubuntu@mira028.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC96nI7dfqxglJOuEna5WiFvFE+vGZgPMy5SE5xay8gFFOedJWT0dHpxo+ewH6o414N1stvesWYJ06e6TQ2ozx1uOShsryZU+i9muFA06dIgtpaT/JpLFxct49brkBbQ8oiWuOnk9TD9eUxvPYwtjeBoNHmXKrd5jT7AObyPIAZmB9ABhe0WBjmQGFyHVAxq01MbWYrYYFsP8mnJV3KvQ7F8KrtoNZXU9UGIVvJ2blmrGv3tNnd6IuHXzi3xvCx/JpkbGEEsWlnVK8h8V4+VlZm4kYvqJVnol8lNRFj7nOiCbbLIB3JlvkOe6PngY4Zb3IvwKndm2/1jSafpYeH2Lrn ubuntu@mira043.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+xOrzZuAdd5+2YlcWPvhdJjJIBcRYFt7QIPXYaWBvP2ncYhZTEL7kM9jAUC6DMFvh3TQrECkouJ0QP+uETQZWdOHcA3Z1am0P/yuaf6cQr31LNyCVjAQjbtStpXzWvrMyEv05ZQ+eTonVbjx1y0nVMFPH8KShNrnXJSn+SArcKt8UPCfvcLH6Ju/R5rfX+YZPMKxuwz9eYerZRsLvXzq/JdR2IGzLGF45nHJ4aD36a07844bfgk6qnLv6LDTAO988FvtOwWHk7jtlMrwXBiqWaGnypcAR9ihKvTL51IiyFFoOvVBDBpDjyA8X3aE0YVvgyEUmCwaEQCVbKs8LcpxT ubuntu@mira079.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQhJKI5vAMk8JO6axI2sHZtNzWtvpV3L1+b9jxqgNyzpFO3CzOdDCbjVd3HQq3/eu0FZLatLcfDvaEuQkcyut6FXGTyQ7w48BwEJCfCaXCqGr9fhd+BPhLFZK7cNexYb0bLPfaJsS9ZHqJMvxFWeRGaMCWphMHIRaoeaMoLXAXzOtLVM82xRO/tTh9t1VPpzH6qdOqdSfxOQOZK8tS7xvxeWhb3TTSBfXqfswHJbKlBx/LDkzO2g/d1sf2MXFFYKVRBRPkrfOBFF9uLY+cXrKGN5rtHwujrxHs+ZLxO7E6JI/fBNIk2xK4DP6Nq76EN0y9JxzBAVfiI7B9LY0qHpx tasks: - chef: null - clock.check: null - install: branch: firefly - ceph: fs: xfs - install.upgrade: osd.0: null - ceph.restart: daemons: - osd.0 - osd.1 - osd.2 - thrashosds: chance_pgnum_grow: 1 chance_pgpnum_fix: 1 timeout: 1200 - ceph.restart: daemons: - mon.a wait-for-healthy: false wait-for-osds-up: true - rados: clients: - client.0 ec_pool: true objects: 500 op_weights: append: 45 delete: 10 read: 45 write: 0 ops: 4000 - ceph.restart: daemons: - mon.b wait-for-healthy: false wait-for-osds-up: true - radosbench: clients: - client.0 time: 1800 - install.upgrade: mon.c: null - ceph.restart: daemons: - mon.c wait-for-healthy: false wait-for-osds-up: true - ceph.wait_for_mon_quorum: - a - b - c - workunit: branch: firefly clients: client.0: - rbd/test_librbd_python.sh - rgw: client.0: idle_timeout: 300 - swift: client.0: rgw_server: client.0 - rados: clients: - client.0 objects: 500 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 teuthology_branch: master 2014-07-01 16:08:32,144.144 INFO:teuthology.run_tasks:Running task internal.save_config... 2014-07-01 16:08:32,144.144 INFO:teuthology.task.internal:Saving configuration 2014-07-01 16:08:32,145.145 INFO:teuthology.run_tasks:Running task internal.check_lock... 2014-07-01 16:08:32,145.145 INFO:teuthology.task.internal:Checking locks... 2014-07-01 16:08:33,044.044 DEBUG:teuthology.task.internal:machine status is {u'sshpubkey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQhJKI5vAMk8JO6axI2sHZtNzWtvpV3L1+b9jxqgNyzpFO3CzOdDCbjVd3HQq3/eu0FZLatLcfDvaEuQkcyut6FXGTyQ7w48BwEJCfCaXCqGr9fhd+BPhLFZK7cNexYb0bLPfaJsS9ZHqJMvxFWeRGaMCWphMHIRaoeaMoLXAXzOtLVM82xRO/tTh9t1VPpzH6qdOqdSfxOQOZK8tS7xvxeWhb3TTSBfXqfswHJbKlBx/LDkzO2g/d1sf2MXFFYKVRBRPkrfOBFF9uLY+cXrKGN5rtHwujrxHs+ZLxO7E6JI/fBNIk2xK4DP6Nq76EN0y9JxzBAVfiI7B9LY0qHpx', u'vpshost': None, u'name': u'ubuntu@mira079.front.sepia.ceph.com', u'type': u'mira', u'description': u'None', u'locked_by': u'loic@dachary.org', u'up': 1, u'mac': u'00:25:90:03:21:6a', u'locked': 1, u'distro': u'ubuntu', u'arch': u'x86_64', u'locked_since': u'2014-07-01T05:48:22'} 2014-07-01 16:08:33,988.988 DEBUG:teuthology.task.internal:machine status is {u'sshpubkey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC96nI7dfqxglJOuEna5WiFvFE+vGZgPMy5SE5xay8gFFOedJWT0dHpxo+ewH6o414N1stvesWYJ06e6TQ2ozx1uOShsryZU+i9muFA06dIgtpaT/JpLFxct49brkBbQ8oiWuOnk9TD9eUxvPYwtjeBoNHmXKrd5jT7AObyPIAZmB9ABhe0WBjmQGFyHVAxq01MbWYrYYFsP8mnJV3KvQ7F8KrtoNZXU9UGIVvJ2blmrGv3tNnd6IuHXzi3xvCx/JpkbGEEsWlnVK8h8V4+VlZm4kYvqJVnol8lNRFj7nOiCbbLIB3JlvkOe6PngY4Zb3IvwKndm2/1jSafpYeH2Lrn', u'vpshost': None, u'name': u'ubuntu@mira028.front.sepia.ceph.com', u'type': u'mira', u'description': u'None', u'locked_by': u'loic@dachary.org', u'up': 1, u'mac': u'00:25:90:09:cc:b4', u'locked': 1, u'distro': u'ubuntu', u'arch': u'x86_64', u'locked_since': u'2014-07-01T01:57:18'} 2014-07-01 16:08:34,839.839 DEBUG:teuthology.task.internal:machine status is {u'sshpubkey': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+xOrzZuAdd5+2YlcWPvhdJjJIBcRYFt7QIPXYaWBvP2ncYhZTEL7kM9jAUC6DMFvh3TQrECkouJ0QP+uETQZWdOHcA3Z1am0P/yuaf6cQr31LNyCVjAQjbtStpXzWvrMyEv05ZQ+eTonVbjx1y0nVMFPH8KShNrnXJSn+SArcKt8UPCfvcLH6Ju/R5rfX+YZPMKxuwz9eYerZRsLvXzq/JdR2IGzLGF45nHJ4aD36a07844bfgk6qnLv6LDTAO988FvtOwWHk7jtlMrwXBiqWaGnypcAR9ihKvTL51IiyFFoOvVBDBpDjyA8X3aE0YVvgyEUmCwaEQCVbKs8LcpxT', u'vpshost': None, u'name': u'ubuntu@mira043.front.sepia.ceph.com', u'type': u'mira', u'description': u'None', u'locked_by': u'loic@dachary.org', u'up': 1, u'mac': u'00:25:90:03:b5:ba', u'locked': 1, u'distro': u'ubuntu', u'arch': u'x86_64', u'locked_since': u'2014-07-01T02:56:03'} 2014-07-01 16:08:34,840.840 INFO:teuthology.run_tasks:Running task internal.connect... 2014-07-01 16:08:34,840.840 INFO:teuthology.task.internal:Opening connections... 2014-07-01 16:08:34,841.841 DEBUG:teuthology.task.internal:connecting to ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:08:34,851.851 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira079.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:08:39,546.546 DEBUG:teuthology.task.internal:connecting to ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:08:39,550.550 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira028.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:08:43,499.499 DEBUG:teuthology.task.internal:connecting to ubuntu@mira043.front.sepia.ceph.com 2014-07-01 16:08:43,504.504 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira043.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:08:47,639.639 INFO:teuthology.task.internal:roles: ubuntu@mira079.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mds.a', 'osd.0', 'osd.1', 'osd.2'] 2014-07-01 16:08:47,639.639 INFO:teuthology.task.internal:roles: ubuntu@mira028.front.sepia.ceph.com - ['osd.3', 'osd.4', 'osd.5', 'mon.c'] 2014-07-01 16:08:47,640.640 INFO:teuthology.task.internal:roles: ubuntu@mira043.front.sepia.ceph.com - ['client.0'] 2014-07-01 16:08:47,640.640 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2014-07-01 16:08:47,640.640 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2014-07-01 16:08:47,640.640 INFO:teuthology.task.internal:Checking for old test directory... 2014-07-01 16:08:47,641.641 INFO:teuthology.orchestra.run.mira028:Running: "test '!' -e /home/ubuntu/cephtest" 2014-07-01 16:08:48,227.227 INFO:teuthology.orchestra.run.mira043:Running: "test '!' -e /home/ubuntu/cephtest" 2014-07-01 16:08:48,666.666 INFO:teuthology.orchestra.run.mira079:Running: "test '!' -e /home/ubuntu/cephtest" 2014-07-01 16:08:49,130.130 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2014-07-01 16:08:49,130.130 INFO:teuthology.task.internal:Checking for old /var/lib/ceph... 2014-07-01 16:08:49,131.131 INFO:teuthology.orchestra.run.mira028:Running: "test '!' -e /var/lib/ceph" 2014-07-01 16:08:49,589.589 INFO:teuthology.orchestra.run.mira043:Running: "test '!' -e /var/lib/ceph" 2014-07-01 16:08:50,090.090 INFO:teuthology.orchestra.run.mira079:Running: "test '!' -e /var/lib/ceph" 2014-07-01 16:08:50,534.534 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2014-07-01 16:08:50,534.534 INFO:teuthology.run_tasks:Running task kernel... 2014-07-01 16:08:50,535.535 INFO:teuthology.task.kernel:config {'mon.a': {'sha1': '8362a1290d075f376ba68521ffb3b42ecaaecfea', 'kdb': True}, 'osd.3': {'sha1': '8362a1290d075f376ba68521ffb3b42ecaaecfea', 'kdb': True}, 'client.0': {'sha1': '8362a1290d075f376ba68521ffb3b42ecaaecfea', 'kdb': True}} 2014-07-01 16:08:50,535.535 DEBUG:teuthology.misc:Using kernel deb sha1 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:50,535.535 DEBUG:teuthology.task.kernel:sha1 for mon.a is 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:50,535.535 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want 8362a1290d075f376ba68521ffb3b42ecaaecfea... 2014-07-01 16:08:50,535.535 INFO:teuthology.orchestra.run.mira079:Running: 'uname -r' 2014-07-01 16:08:51,230.230 DEBUG:teuthology.task.kernel:current kernel version is 3.16.0-rc2-ceph-00018-g8362a12 2014-07-01 16:08:51,230.230 DEBUG:teuthology.task.kernel:extracting sha1, 3.16.0-rc2-ceph-00018-g8362a12 -> 8362a12 2014-07-01 16:08:51,230.230 DEBUG:teuthology.task.kernel:extracted sha1 matches, do not need to install 2014-07-01 16:08:51,230.230 DEBUG:teuthology.misc:Using kernel deb sha1 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:51,230.230 DEBUG:teuthology.task.kernel:sha1 for osd.3 is 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:51,230.230 INFO:teuthology.task.kernel:Checking kernel version of osd.3, want 8362a1290d075f376ba68521ffb3b42ecaaecfea... 2014-07-01 16:08:51,231.231 INFO:teuthology.orchestra.run.mira028:Running: 'uname -r' 2014-07-01 16:08:51,690.690 DEBUG:teuthology.task.kernel:current kernel version is 3.16.0-rc2-ceph-00018-g8362a12 2014-07-01 16:08:51,690.690 DEBUG:teuthology.task.kernel:extracting sha1, 3.16.0-rc2-ceph-00018-g8362a12 -> 8362a12 2014-07-01 16:08:51,690.690 DEBUG:teuthology.task.kernel:extracted sha1 matches, do not need to install 2014-07-01 16:08:51,690.690 DEBUG:teuthology.misc:Using kernel deb sha1 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:51,690.690 DEBUG:teuthology.task.kernel:sha1 for client.0 is 8362a1290d075f376ba68521ffb3b42ecaaecfea 2014-07-01 16:08:51,690.690 INFO:teuthology.task.kernel:Checking kernel version of client.0, want 8362a1290d075f376ba68521ffb3b42ecaaecfea... 2014-07-01 16:08:51,690.690 INFO:teuthology.orchestra.run.mira043:Running: 'uname -r' 2014-07-01 16:08:52,153.153 DEBUG:teuthology.task.kernel:current kernel version is 3.16.0-rc2-ceph-00018-g8362a12 2014-07-01 16:08:52,153.153 DEBUG:teuthology.task.kernel:extracting sha1, 3.16.0-rc2-ceph-00018-g8362a12 -> 8362a12 2014-07-01 16:08:52,153.153 DEBUG:teuthology.task.kernel:extracted sha1 matches, do not need to install 2014-07-01 16:08:52,153.153 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2014-07-01 16:08:52,153.153 INFO:teuthology.orchestra.run.mira079:Running: 'echo ttyS2 | sudo tee /sys/module/kgdboc/parameters/kgdboc' 2014-07-01 16:08:52,642.642 INFO:teuthology.orchestra.run.mira079.stdout:ttyS2 2014-07-01 16:08:52,674.674 INFO:teuthology.task.kernel:Enabling kdb on osd.3... 2014-07-01 16:08:52,674.674 INFO:teuthology.orchestra.run.mira028:Running: 'echo ttyS2 | sudo tee /sys/module/kgdboc/parameters/kgdboc' 2014-07-01 16:08:53,164.164 INFO:teuthology.orchestra.run.mira028.stdout:ttyS2 2014-07-01 16:08:53,189.189 INFO:teuthology.task.kernel:Enabling kdb on client.0... 2014-07-01 16:08:53,189.189 INFO:teuthology.orchestra.run.mira043:Running: 'echo ttyS2 | sudo tee /sys/module/kgdboc/parameters/kgdboc' 2014-07-01 16:08:53,887.887 INFO:teuthology.orchestra.run.mira043.stdout:ttyS2 2014-07-01 16:08:53,910.910 INFO:teuthology.run_tasks:Running task internal.base... 2014-07-01 16:08:53,910.910 INFO:teuthology.task.internal:Creating test directory... 2014-07-01 16:08:53,910.910 INFO:teuthology.orchestra.run.mira028:Running: 'mkdir -m0755 -- /home/ubuntu/cephtest' 2014-07-01 16:08:54,367.367 INFO:teuthology.orchestra.run.mira043:Running: 'mkdir -m0755 -- /home/ubuntu/cephtest' 2014-07-01 16:08:54,826.826 INFO:teuthology.orchestra.run.mira079:Running: 'mkdir -m0755 -- /home/ubuntu/cephtest' 2014-07-01 16:08:55,368.368 INFO:teuthology.run_tasks:Running task internal.archive... 2014-07-01 16:08:55,368.368 INFO:teuthology.task.internal:Creating archive directory... 2014-07-01 16:08:55,369.369 INFO:teuthology.orchestra.run.mira028:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive' 2014-07-01 16:08:55,869.869 INFO:teuthology.orchestra.run.mira043:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive' 2014-07-01 16:08:56,533.533 INFO:teuthology.orchestra.run.mira079:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive' 2014-07-01 16:08:57,055.055 INFO:teuthology.run_tasks:Running task internal.coredump... 2014-07-01 16:08:57,055.055 INFO:teuthology.task.internal:Enabling coredump saving... 2014-07-01 16:08:57,056.056 INFO:teuthology.orchestra.run.mira028:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core' 2014-07-01 16:08:57,581.581 INFO:teuthology.orchestra.run.mira043:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core' 2014-07-01 16:08:57,608.608 INFO:teuthology.orchestra.run.mira028.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2014-07-01 16:08:58,213.213 INFO:teuthology.orchestra.run.mira079:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core' 2014-07-01 16:08:58,215.215 INFO:teuthology.orchestra.run.mira043.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2014-07-01 16:08:58,669.669 INFO:teuthology.orchestra.run.mira079.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2014-07-01 16:08:58,670.670 INFO:teuthology.run_tasks:Running task internal.sudo... 2014-07-01 16:08:58,670.670 INFO:teuthology.task.internal:Configuring sudo... 2014-07-01 16:08:58,670.670 INFO:teuthology.orchestra.run.mira028:Running: "sudo sed -i.orig.teuthology -e 's/^\\([^#]*\\) \\(requiretty\\)/\\1 !\\2/g' -e 's/^\\([^#]*\\) !\\(visiblepw\\)/\\1 \\2/g' /etc/sudoers" 2014-07-01 16:08:59,260.260 INFO:teuthology.orchestra.run.mira043:Running: "sudo sed -i.orig.teuthology -e 's/^\\([^#]*\\) \\(requiretty\\)/\\1 !\\2/g' -e 's/^\\([^#]*\\) !\\(visiblepw\\)/\\1 \\2/g' /etc/sudoers" 2014-07-01 16:08:59,887.887 INFO:teuthology.orchestra.run.mira079:Running: "sudo sed -i.orig.teuthology -e 's/^\\([^#]*\\) \\(requiretty\\)/\\1 !\\2/g' -e 's/^\\([^#]*\\) !\\(visiblepw\\)/\\1 \\2/g' /etc/sudoers" 2014-07-01 16:09:00,361.361 INFO:teuthology.run_tasks:Running task internal.syslog... 2014-07-01 16:09:00,361.361 INFO:teuthology.run_tasks:Running task internal.timer... 2014-07-01 16:09:00,362.362 INFO:teuthology.task.internal:Starting timer... 2014-07-01 16:09:00,362.362 INFO:teuthology.run_tasks:Running task chef... 2014-07-01 16:09:00,363.363 INFO:teuthology.task.chef:Running chef-solo... 2014-07-01 16:09:00,363.363 INFO:teuthology.orchestra.run.mira028:Running: "wget -O- 'http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD' | sh -x" 2014-07-01 16:09:00,955.955 INFO:teuthology.orchestra.run.mira043:Running: "wget -O- 'http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD' | sh -x" 2014-07-01 16:09:01,014.014 INFO:teuthology.orchestra.run.mira028.stderr:--2014-07-01 07:09:00-- http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD 2014-07-01 16:09:01,019.019 INFO:teuthology.orchestra.run.mira028.stderr:Resolving ceph.com (ceph.com)... 208.113.241.137, 2607:f298:4:147::b05:fe2a 2014-07-01 16:09:01,020.020 INFO:teuthology.orchestra.run.mira028.stderr:Connecting to ceph.com (ceph.com)|208.113.241.137|:80... connected. 2014-07-01 16:09:01,103.103 INFO:teuthology.orchestra.run.mira028.stderr:HTTP request sent, awaiting response... 200 OK 2014-07-01 16:09:01,103.103 INFO:teuthology.orchestra.run.mira028.stderr:Length: unspecified [text/plain] 2014-07-01 16:09:01,104.104 INFO:teuthology.orchestra.run.mira028.stderr:Saving to: `STDOUT' 2014-07-01 16:09:01,105.105 INFO:teuthology.orchestra.run.mira028.stderr: 2014-07-01 16:09:01,106.106 INFO:teuthology.orchestra.run.mira028.stderr: 0K ...+ set -e 2014-07-01 16:09:01,106.106 INFO:teuthology.orchestra.run.mira028.stderr:+ dirname sh 2014-07-01 16:09:01,187.187 INFO:teuthology.orchestra.run.mira028.stderr: 89.5M=0s 2014-07-01 16:09:01,187.187 INFO:teuthology.orchestra.run.mira028.stderr: 2014-07-01 16:09:01,187.187 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:09:00 (89.5 MB/s) - written to stdout [3313] 2014-07-01 16:09:01,188.188 INFO:teuthology.orchestra.run.mira028.stderr: 2014-07-01 16:09:01,188.188 INFO:teuthology.orchestra.run.mira028.stderr:+ cd . 2014-07-01 16:09:01,188.188 INFO:teuthology.orchestra.run.mira028.stderr:+ python -c import platform; distro = platform.dist()[0]; print distro.lower().rstrip() 2014-07-01 16:09:01,188.188 INFO:teuthology.orchestra.run.mira028.stderr:+ distro=ubuntu 2014-07-01 16:09:01,189.189 INFO:teuthology.orchestra.run.mira028.stderr:+ python -c import platform; distro = platform.dist()[1]; print distro 2014-07-01 16:09:01,189.189 INFO:teuthology.orchestra.run.mira028.stderr:+ distroversion=12.04 2014-07-01 16:09:01,189.189 INFO:teuthology.orchestra.run.mira028.stderr:+ echo ubuntu 2014-07-01 16:09:01,189.189 INFO:teuthology.orchestra.run.mira028.stderr:+ grep -ic -e centos -e redhat -e fedora 2014-07-01 16:09:01,190.190 INFO:teuthology.orchestra.run.mira028.stderr:+ true 2014-07-01 16:09:01,190.190 INFO:teuthology.orchestra.run.mira028.stderr:+ centos=0 2014-07-01 16:09:01,190.190 INFO:teuthology.orchestra.run.mira028.stderr:+ echo ubuntu 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ grep -ic -e ubuntu -e debian 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ ubuntu=1 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ echo 12.04 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ grep -ic 14\.04 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ true 2014-07-01 16:09:01,191.191 INFO:teuthology.orchestra.run.mira028.stderr:+ trusty=0 2014-07-01 16:09:01,192.192 INFO:teuthology.orchestra.run.mira028.stderr:+ [ 1 -gt 0 ] 2014-07-01 16:09:01,192.192 INFO:teuthology.orchestra.run.mira028.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:01,192.192 INFO:teuthology.orchestra.run.mira028.stderr:+ nodepends=--no-install-recommends 2014-07-01 16:09:01,192.192 INFO:teuthology.orchestra.run.mira028.stderr:+ dpkg --status -- git 2014-07-01 16:09:01,192.192 INFO:teuthology.orchestra.run.mira028.stderr:+ sed -n s/^Status: //p 2014-07-01 16:09:01,237.237 INFO:teuthology.orchestra.run.mira028.stderr:+ [ install ok installed != install ok installed ] 2014-07-01 16:09:01,238.238 INFO:teuthology.orchestra.run.mira028.stderr:+ [ -n ] 2014-07-01 16:09:01,238.238 INFO:teuthology.orchestra.run.mira028.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:01,238.238 INFO:teuthology.orchestra.run.mira028.stderr:+ mktemp -d --tmpdir solo-from-scratch.XXXXXXXXXXXX 2014-07-01 16:09:01,238.238 INFO:teuthology.orchestra.run.mira028.stderr:+ SCRATCH=/tmp/solo-from-scratch.EiUJGCEOEa09 2014-07-01 16:09:01,238.238 INFO:teuthology.orchestra.run.mira028.stderr:+ cd /tmp/solo-from-scratch.EiUJGCEOEa09 2014-07-01 16:09:01,239.239 INFO:teuthology.orchestra.run.mira028.stderr:+ trap cleanup INT TERM EXIT 2014-07-01 16:09:01,239.239 INFO:teuthology.orchestra.run.mira028.stderr:+ git init 2014-07-01 16:09:01,321.321 INFO:teuthology.orchestra.run.mira028.stdout:Initialized empty Git repository in /tmp/solo-from-scratch.EiUJGCEOEa09/.git/ 2014-07-01 16:09:01,325.325 INFO:teuthology.orchestra.run.mira028.stderr:+ git pull git://ceph.com/ceph-qa-chef.git 2014-07-01 16:09:01,587.587 INFO:teuthology.orchestra.run.mira079:Running: "wget -O- 'http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD' | sh -x" 2014-07-01 16:09:01,635.635 INFO:teuthology.orchestra.run.mira043.stderr:--2014-07-01 07:09:01-- http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD 2014-07-01 16:09:01,635.635 INFO:teuthology.orchestra.run.mira043.stderr:Resolving ceph.com (ceph.com)... 208.113.241.137, 2607:f298:4:147::b05:fe2a 2014-07-01 16:09:01,635.635 INFO:teuthology.orchestra.run.mira043.stderr:Connecting to ceph.com (ceph.com)|208.113.241.137|:80... connected. 2014-07-01 16:09:01,770.770 INFO:teuthology.orchestra.run.mira043.stderr:HTTP request sent, awaiting response... 200 OK 2014-07-01 16:09:01,771.771 INFO:teuthology.orchestra.run.mira043.stderr:Length: unspecified [text/plain] 2014-07-01 16:09:01,771.771 INFO:teuthology.orchestra.run.mira043.stderr:Saving to: `STDOUT' 2014-07-01 16:09:01,771.771 INFO:teuthology.orchestra.run.mira043.stderr: 2014-07-01 16:09:01,772.772 INFO:teuthology.orchestra.run.mira043.stderr: 0K ...+ set -e 2014-07-01 16:09:01,772.772 INFO:teuthology.orchestra.run.mira043.stderr:+ dirname sh 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr: 87.1M=0s 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr: 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr:2014-07-01 07:09:01 (87.1 MB/s) - written to stdout [3313] 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr: 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr:+ cd . 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr:+ python -c import platform; distro = platform.dist()[0]; print distro.lower().rstrip() 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr:+ distro=ubuntu 2014-07-01 16:09:01,814.814 INFO:teuthology.orchestra.run.mira043.stderr:+ python -c import platform; distro = platform.dist()[1]; print distro 2014-07-01 16:09:01,827.827 INFO:teuthology.orchestra.run.mira043.stderr:+ distroversion=12.04 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ echo ubuntu 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ grep -ic -e centos -e redhat -e fedora 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ true 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ centos=0 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ echo ubuntu 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ grep -ic -e ubuntu -e debian 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ ubuntu=1 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ echo 12.04 2014-07-01 16:09:01,863.863 INFO:teuthology.orchestra.run.mira043.stderr:+ grep -ic 14\.04 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ true 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ trusty=0 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ [ 1 -gt 0 ] 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ nodepends=--no-install-recommends 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ dpkg --status -- git 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ sed -n s/^Status: //p 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ [ install ok installed != install ok installed ] 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ [ -n ] 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:01,864.864 INFO:teuthology.orchestra.run.mira043.stderr:+ mktemp -d --tmpdir solo-from-scratch.XXXXXXXXXXXX 2014-07-01 16:09:01,887.887 INFO:teuthology.orchestra.run.mira043.stderr:+ SCRATCH=/tmp/solo-from-scratch.t9TrCQTgYTKi 2014-07-01 16:09:01,887.887 INFO:teuthology.orchestra.run.mira043.stderr:+ cd /tmp/solo-from-scratch.t9TrCQTgYTKi 2014-07-01 16:09:02,000.000 INFO:teuthology.orchestra.run.mira043.stderr:+ trap cleanup INT TERM EXIT 2014-07-01 16:09:02,000.000 INFO:teuthology.orchestra.run.mira043.stderr:+ git init 2014-07-01 16:09:02,000.000 INFO:teuthology.orchestra.run.mira043.stderr:+ git pull git://ceph.com/ceph-qa-chef.git 2014-07-01 16:09:02,001.001 INFO:teuthology.orchestra.run.mira043.stdout:Initialized empty Git repository in /tmp/solo-from-scratch.t9TrCQTgYTKi/.git/ 2014-07-01 16:09:02,128.128 INFO:teuthology.orchestra.run.mira079.stderr:--2014-07-01 07:09:02-- http://ceph.com/git/?p=ceph-qa-chef.git;a=blob_plain;f=solo/solo-from-scratch;hb=HEAD 2014-07-01 16:09:02,131.131 INFO:teuthology.orchestra.run.mira079.stderr:Resolving ceph.com (ceph.com)... 208.113.241.137, 2607:f298:4:147::b05:fe2a 2014-07-01 16:09:02,132.132 INFO:teuthology.orchestra.run.mira079.stderr:Connecting to ceph.com (ceph.com)|208.113.241.137|:80... connected. 2014-07-01 16:09:02,266.266 INFO:teuthology.orchestra.run.mira079.stderr:HTTP request sent, awaiting response... 200 OK 2014-07-01 16:09:02,267.267 INFO:teuthology.orchestra.run.mira079.stderr:Length: unspecified [text/plain] 2014-07-01 16:09:02,267.267 INFO:teuthology.orchestra.run.mira079.stderr:Saving to: `STDOUT' 2014-07-01 16:09:02,267.267 INFO:teuthology.orchestra.run.mira079.stderr: 2014-07-01 16:09:02,269.269 INFO:teuthology.orchestra.run.mira079.stderr: 0K ...+ set -e 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr:+ dirname sh 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr: 87.6M=0s 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr: 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:09:02 (87.6 MB/s) - written to stdout [3313] 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr: 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr:+ cd . 2014-07-01 16:09:02,278.278 INFO:teuthology.orchestra.run.mira079.stderr:+ python -c import platform; distro = platform.dist()[0]; print distro.lower().rstrip() 2014-07-01 16:09:02,302.302 INFO:teuthology.orchestra.run.mira079.stderr:+ distro=ubuntu 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ python -c import platform; distro = platform.dist()[1]; print distro 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ distroversion=12.04 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ echo ubuntu 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ grep -ic -e centos -e redhat -e fedora 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ true 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ centos=0 2014-07-01 16:09:02,363.363 INFO:teuthology.orchestra.run.mira079.stderr:+ echo ubuntu 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ grep -ic -e ubuntu -e debian 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ ubuntu=1 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ echo 12.04 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ grep -ic 14\.04 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ true 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ trusty=0 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ [ 1 -gt 0 ] 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ nodepends=--no-install-recommends 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ dpkg --status -- git 2014-07-01 16:09:02,364.364 INFO:teuthology.orchestra.run.mira079.stderr:+ sed -n s/^Status: //p 2014-07-01 16:09:02,375.375 INFO:teuthology.orchestra.run.mira079.stderr:+ [ install ok installed != install ok installed ] 2014-07-01 16:09:02,376.376 INFO:teuthology.orchestra.run.mira079.stderr:+ [ -n ] 2014-07-01 16:09:02,376.376 INFO:teuthology.orchestra.run.mira079.stderr:+ [ 0 -gt 0 ] 2014-07-01 16:09:02,377.377 INFO:teuthology.orchestra.run.mira079.stderr:+ mktemp -d --tmpdir solo-from-scratch.XXXXXXXXXXXX 2014-07-01 16:09:02,399.399 INFO:teuthology.orchestra.run.mira079.stderr:+ SCRATCH=/tmp/solo-from-scratch.nJK73qjNeLkS 2014-07-01 16:09:02,400.400 INFO:teuthology.orchestra.run.mira079.stderr:+ cd /tmp/solo-from-scratch.nJK73qjNeLkS 2014-07-01 16:09:02,497.497 INFO:teuthology.orchestra.run.mira079.stderr:+ trap cleanup INT TERM EXIT 2014-07-01 16:09:02,497.497 INFO:teuthology.orchestra.run.mira079.stderr:+ git init 2014-07-01 16:09:02,511.511 INFO:teuthology.orchestra.run.mira079.stdout:Initialized empty Git repository in /tmp/solo-from-scratch.nJK73qjNeLkS/.git/ 2014-07-01 16:09:02,511.511 INFO:teuthology.orchestra.run.mira079.stderr:+ git pull git://ceph.com/ceph-qa-chef.git 2014-07-01 16:09:02,879.879 INFO:teuthology.orchestra.run.mira028.stderr:From git://ceph.com/ceph-qa-chef 2014-07-01 16:09:02,880.880 INFO:teuthology.orchestra.run.mira028.stderr: * branch HEAD -> FETCH_HEAD 2014-07-01 16:09:02,942.942 INFO:teuthology.orchestra.run.mira028.stderr:+ ./solo/run 2014-07-01 16:09:03,811.811 INFO:teuthology.orchestra.run.mira043.stderr:From git://ceph.com/ceph-qa-chef 2014-07-01 16:09:03,812.812 INFO:teuthology.orchestra.run.mira043.stderr: * branch HEAD -> FETCH_HEAD 2014-07-01 16:09:03,867.867 INFO:teuthology.orchestra.run.mira043.stderr:+ ./solo/run 2014-07-01 16:09:04,122.122 INFO:teuthology.orchestra.run.mira079.stderr:From git://ceph.com/ceph-qa-chef 2014-07-01 16:09:04,122.122 INFO:teuthology.orchestra.run.mira079.stderr: * branch HEAD -> FETCH_HEAD 2014-07-01 16:09:04,195.195 INFO:teuthology.orchestra.run.mira079.stderr:+ ./solo/run 2014-07-01 16:09:05,842.842 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:05-07:00] INFO: Forking chef instance to converge... 2014-07-01 16:09:05,845.845 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:05-07:00] WARN: 2014-07-01 16:09:05,845.845 INFO:teuthology.orchestra.run.mira028.stdout:* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2014-07-01 16:09:05,846.846 INFO:teuthology.orchestra.run.mira028.stdout:SSL validation of HTTPS requests is disabled. HTTPS connections are still 2014-07-01 16:09:05,846.846 INFO:teuthology.orchestra.run.mira028.stdout:encrypted, but chef is not able to detect forged replies or man in the middle 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout:attacks. 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout:To fix this issue add an entry like this to your configuration file: 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout:``` 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout: # Verify all HTTPS connections (recommended) 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout: ssl_verify_mode :verify_peer 2014-07-01 16:09:05,847.847 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: # OR, Verify only connections to chef-server 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: verify_api_cert true 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:``` 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:To check your SSL configuration, or troubleshoot errors, you can use the 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:`knife ssl check` command like so: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:``` 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: knife ssl check -c solo.rb 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:``` 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout: 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:05-07:00] INFO: *** Chef 11.12.8 *** 2014-07-01 16:09:05,848.848 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:05-07:00] INFO: Chef-client pid: 3119 2014-07-01 16:09:06,168.168 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:05-07:00] INFO: Forking chef instance to converge... 2014-07-01 16:09:06,168.168 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:05-07:00] INFO: *** Chef 11.10.4 *** 2014-07-01 16:09:06,168.168 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:05-07:00] INFO: Chef-client pid: 2894 2014-07-01 16:09:06,397.397 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:06-07:00] INFO: Forking chef instance to converge... 2014-07-01 16:09:06,400.400 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:06-07:00] INFO: *** Chef 11.6.2 *** 2014-07-01 16:09:10,548.548 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Setting the run_list to ["recipe[ceph-qa::default]"] from JSON 2014-07-01 16:09:10,549.549 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Run List is [recipe[ceph-qa::default]] 2014-07-01 16:09:10,549.549 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Run List expands to [ceph-qa::default] 2014-07-01 16:09:10,549.549 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Starting Chef Run for mira043.front.sepia.ceph.com 2014-07-01 16:09:10,550.550 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Running start handlers 2014-07-01 16:09:10,550.550 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Start handlers complete. 2014-07-01 16:09:10,592.592 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Processing file[/etc/security/limits.d/ubuntu.conf] action create (ceph-qa::default line 8) 2014-07-01 16:09:10,601.601 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:10-07:00] INFO: Processing execute[remove ceph packages] action run (ceph-qa::ubuntu line 9) 2014-07-01 16:09:10,694.694 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Setting the run_list to ["recipe[ceph-qa::default]"] from JSON 2014-07-01 16:09:10,702.702 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Run List is [recipe[ceph-qa::default]] 2014-07-01 16:09:10,702.702 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Run List expands to [ceph-qa::default] 2014-07-01 16:09:10,703.703 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Starting Chef Run for mira079.front.sepia.ceph.com 2014-07-01 16:09:10,703.703 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Running start handlers 2014-07-01 16:09:10,704.704 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Start handlers complete. 2014-07-01 16:09:10,746.746 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Processing file[/etc/security/limits.d/ubuntu.conf] action create (ceph-qa::default line 8) 2014-07-01 16:09:10,747.747 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:10-07:00] INFO: Processing execute[remove ceph packages] action run (ceph-qa::ubuntu line 9) 2014-07-01 16:09:13,225.225 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Setting the run_list to ["recipe[ceph-qa::default]"] from CLI options 2014-07-01 16:09:13,231.231 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Run List is [recipe[ceph-qa::default]] 2014-07-01 16:09:13,231.231 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Run List expands to [ceph-qa::default] 2014-07-01 16:09:13,231.231 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Starting Chef Run for mira028.front.sepia.ceph.com 2014-07-01 16:09:13,233.233 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Running start handlers 2014-07-01 16:09:13,233.233 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Start handlers complete. 2014-07-01 16:09:13,266.266 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing file[/etc/security/limits.d/ubuntu.conf] action create (ceph-qa::default line 8) 2014-07-01 16:09:13,270.270 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[remove ceph packages] action run (ceph-qa::ubuntu line 9) 2014-07-01 16:09:13,482.482 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove ceph packages] ran successfully 2014-07-01 16:09:13,484.484 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[remove /etc/ceph] action run (ceph-qa::ubuntu line 12) 2014-07-01 16:09:13,525.525 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove /etc/ceph] ran successfully 2014-07-01 16:09:13,525.525 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[remove old source entries] action run (ceph-qa::ubuntu line 15) 2014-07-01 16:09:13,560.560 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove old source entries] ran successfully 2014-07-01 16:09:13,561.561 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing cookbook_file[/etc/apt/sources.list] action create (ceph-qa::ubuntu line 41) 2014-07-01 16:09:13,693.693 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove ceph packages] ran successfully 2014-07-01 16:09:13,693.693 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[remove /etc/ceph] action run (ceph-qa::ubuntu line 12) 2014-07-01 16:09:13,776.776 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove /etc/ceph] ran successfully 2014-07-01 16:09:13,777.777 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[remove old source entries] action run (ceph-qa::ubuntu line 15) 2014-07-01 16:09:13,839.839 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[remove old source entries] ran successfully 2014-07-01 16:09:13,841.841 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing cookbook_file[/etc/apt/sources.list] action create (ceph-qa::ubuntu line 41) 2014-07-01 16:09:13,901.901 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing cookbook_file[/etc/cron.weekly/kernel-clean] action create (ceph-qa::ubuntu line 66) 2014-07-01 16:09:13,935.935 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[Restarting Cron] action run (ceph-qa::ubuntu line 72) 2014-07-01 16:09:13,942.942 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing cookbook_file[/etc/cron.weekly/kernel-clean] action create (ceph-qa::ubuntu line 66) 2014-07-01 16:09:13,942.942 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[Restarting Cron] action run (ceph-qa::ubuntu line 72) 2014-07-01 16:09:13,946.946 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[Restarting Cron] ran successfully 2014-07-01 16:09:13,946.946 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[add autobuild gpg key to apt] action run (ceph-qa::ubuntu line 87) 2014-07-01 16:09:14,034.034 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: execute[Restarting Cron] ran successfully 2014-07-01 16:09:14,034.034 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:13-07:00] INFO: Processing execute[add autobuild gpg key to apt] action run (ceph-qa::ubuntu line 87) 2014-07-01 16:09:14,273.273 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: execute[remove ceph packages] ran successfully 2014-07-01 16:09:14,274.274 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing execute[remove /etc/ceph] action run (ceph-qa::ubuntu line 12) 2014-07-01 16:09:14,311.311 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: execute[remove /etc/ceph] ran successfully 2014-07-01 16:09:14,311.311 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing execute[remove old source entries] action run (ceph-qa::ubuntu line 15) 2014-07-01 16:09:14,346.346 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: execute[remove old source entries] ran successfully 2014-07-01 16:09:14,347.347 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing cookbook_file[/etc/apt/sources.list] action create (ceph-qa::ubuntu line 41) 2014-07-01 16:09:14,419.419 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing cookbook_file[/etc/cron.weekly/kernel-clean] action create (ceph-qa::ubuntu line 66) 2014-07-01 16:09:14,451.451 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing execute[Restarting Cron] action run (ceph-qa::ubuntu line 72) 2014-07-01 16:09:14,498.498 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: execute[Restarting Cron] ran successfully 2014-07-01 16:09:14,498.498 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:14-07:00] INFO: Processing execute[add autobuild gpg key to apt] action run (ceph-qa::ubuntu line 87) 2014-07-01 16:09:16,751.751 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: execute[add autobuild gpg key to apt] ran successfully 2014-07-01 16:09:16,751.751 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: Processing file[/etc/apt/sources.list.d/ceph-extras.list] action create (ceph-qa::ubuntu line 96) 2014-07-01 16:09:16,753.753 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] created file /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:16,813.813 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] updated file contents /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:16,813.813 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] owner changed to 0 2014-07-01 16:09:16,815.815 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] group changed to 0 2014-07-01 16:09:16,815.815 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] mode changed to 644 2014-07-01 16:09:16,817.817 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:16-07:00] INFO: Processing execute[add release gpg key to apt] action run (ceph-qa::ubuntu line 104) 2014-07-01 16:09:17,150.150 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: execute[add autobuild gpg key to apt] ran successfully 2014-07-01 16:09:17,152.152 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: Processing file[/etc/apt/sources.list.d/ceph-extras.list] action create (ceph-qa::ubuntu line 96) 2014-07-01 16:09:17,156.156 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] created file /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:17,161.161 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] updated file contents /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:17,164.164 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] owner changed to 0 2014-07-01 16:09:17,164.164 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] group changed to 0 2014-07-01 16:09:17,165.165 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] mode changed to 644 2014-07-01 16:09:17,165.165 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:17-07:00] INFO: Processing execute[add release gpg key to apt] action run (ceph-qa::ubuntu line 104) 2014-07-01 16:09:17,560.560 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: execute[add autobuild gpg key to apt] ran successfully 2014-07-01 16:09:17,561.561 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: Processing file[/etc/apt/sources.list.d/ceph-extras.list] action create (ceph-qa::ubuntu line 96) 2014-07-01 16:09:17,562.562 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] created file /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:17,576.576 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] updated file contents /etc/apt/sources.list.d/ceph-extras.list 2014-07-01 16:09:17,579.579 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] owner changed to 0 2014-07-01 16:09:17,580.580 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] group changed to 0 2014-07-01 16:09:17,580.580 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: file[/etc/apt/sources.list.d/ceph-extras.list] mode changed to 644 2014-07-01 16:09:17,581.581 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:17-07:00] INFO: Processing execute[add release gpg key to apt] action run (ceph-qa::ubuntu line 104) 2014-07-01 16:09:19,433.433 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:19-07:00] INFO: execute[add release gpg key to apt] ran successfully 2014-07-01 16:09:19,433.433 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:19-07:00] INFO: Processing package[python-pip] action install (ceph-qa::ubuntu line 112) 2014-07-01 16:09:19,567.567 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:19-07:00] INFO: execute[add release gpg key to apt] ran successfully 2014-07-01 16:09:19,568.568 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:19-07:00] INFO: Processing package[python-pip] action install (ceph-qa::ubuntu line 112) 2014-07-01 16:09:19,910.910 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:19-07:00] INFO: execute[add release gpg key to apt] ran successfully 2014-07-01 16:09:19,910.910 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:19-07:00] INFO: Processing package[python-pip] action install (ceph-qa::ubuntu line 112) 2014-07-01 16:09:21,971.971 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:21-07:00] INFO: Processing package[python-virtualenv] action install (ceph-qa::ubuntu line 113) 2014-07-01 16:09:21,993.993 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:21-07:00] INFO: Processing package[python-virtualenv] action install (ceph-qa::ubuntu line 113) 2014-07-01 16:09:22,032.032 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:21-07:00] INFO: Processing package[python-dev] action install (ceph-qa::ubuntu line 114) 2014-07-01 16:09:22,085.085 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:21-07:00] INFO: Processing package[python-dev] action install (ceph-qa::ubuntu line 114) 2014-07-01 16:09:22,091.091 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:21-07:00] INFO: Processing package[libevent-dev] action install (ceph-qa::ubuntu line 115) 2014-07-01 16:09:22,151.151 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[perl] action install (ceph-qa::ubuntu line 118) 2014-07-01 16:09:22,178.178 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[libevent-dev] action install (ceph-qa::ubuntu line 115) 2014-07-01 16:09:22,208.208 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[libwww-perl] action install (ceph-qa::ubuntu line 119) 2014-07-01 16:09:22,264.264 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[perl] action install (ceph-qa::ubuntu line 118) 2014-07-01 16:09:22,271.271 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/home/ubuntu/.cpan/CPAN/] action create (ceph-qa::ubuntu line 121) 2014-07-01 16:09:22,351.351 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[libwww-perl] action install (ceph-qa::ubuntu line 119) 2014-07-01 16:09:22,415.415 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[python-virtualenv] action install (ceph-qa::ubuntu line 113) 2014-07-01 16:09:22,435.435 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/home/ubuntu/.cpan/CPAN/] action create (ceph-qa::ubuntu line 121) 2014-07-01 16:09:22,451.451 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/home/ubuntu/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 127) 2014-07-01 16:09:22,465.465 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/root/.cpan/CPAN/] action create (ceph-qa::ubuntu line 134) 2014-07-01 16:09:22,475.475 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[python-dev] action install (ceph-qa::ubuntu line 114) 2014-07-01 16:09:22,482.482 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/root/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 140) 2014-07-01 16:09:22,488.488 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/preferences.d/ceph.pref] action create (ceph-qa::radosgw line 4) 2014-07-01 16:09:22,489.489 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/sources.list.d/radosgw.list] action create (ceph-qa::radosgw line 16) 2014-07-01 16:09:22,492.492 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] created file /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:22,525.525 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/home/ubuntu/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 127) 2014-07-01 16:09:22,526.526 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] updated file contents /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:22,526.526 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] owner changed to 0 2014-07-01 16:09:22,527.527 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] group changed to 0 2014-07-01 16:09:22,527.527 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] mode changed to 644 2014-07-01 16:09:22,528.528 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing execute[apt-get update] action run (ceph-qa::radosgw line 54) 2014-07-01 16:09:22,529.529 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[libevent-dev] action install (ceph-qa::ubuntu line 115) 2014-07-01 16:09:22,558.558 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/root/.cpan/CPAN/] action create (ceph-qa::ubuntu line 134) 2014-07-01 16:09:22,584.584 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[perl] action install (ceph-qa::ubuntu line 118) 2014-07-01 16:09:22,605.605 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/root/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 140) 2014-07-01 16:09:22,640.640 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing package[libwww-perl] action install (ceph-qa::ubuntu line 119) 2014-07-01 16:09:22,696.696 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/home/ubuntu/.cpan/CPAN/] action create (ceph-qa::ubuntu line 121) 2014-07-01 16:09:22,712.712 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/preferences.d/ceph.pref] action create (ceph-qa::radosgw line 4) 2014-07-01 16:09:22,716.716 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/sources.list.d/radosgw.list] action create (ceph-qa::radosgw line 16) 2014-07-01 16:09:22,717.717 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] created file /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:22,720.720 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] updated file contents /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:22,721.721 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] owner changed to 0 2014-07-01 16:09:22,721.721 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] group changed to 0 2014-07-01 16:09:22,722.722 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] mode changed to 644 2014-07-01 16:09:22,756.756 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing execute[apt-get update] action run (ceph-qa::radosgw line 54) 2014-07-01 16:09:22,972.972 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/home/ubuntu/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 127) 2014-07-01 16:09:22,978.978 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing directory[/root/.cpan/CPAN/] action create (ceph-qa::ubuntu line 134) 2014-07-01 16:09:22,989.989 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing cookbook_file[/root/.cpan/CPAN/MyConfig.pm] action create (ceph-qa::ubuntu line 140) 2014-07-01 16:09:22,989.989 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/preferences.d/ceph.pref] action create (ceph-qa::radosgw line 4) 2014-07-01 16:09:22,995.995 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing file[/etc/apt/sources.list.d/radosgw.list] action create (ceph-qa::radosgw line 16) 2014-07-01 16:09:23,003.003 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] created file /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:23,005.005 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] updated file contents /etc/apt/sources.list.d/radosgw.list 2014-07-01 16:09:23,005.005 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] owner changed to 0 2014-07-01 16:09:23,006.006 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] group changed to 0 2014-07-01 16:09:23,006.006 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: file[/etc/apt/sources.list.d/radosgw.list] mode changed to 644 2014-07-01 16:09:23,200.200 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:22-07:00] INFO: Processing execute[apt-get update] action run (ceph-qa::radosgw line 54) 2014-07-01 16:09:25,043.043 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:24-07:00] INFO: execute[apt-get update] ran successfully 2014-07-01 16:09:25,044.044 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:24-07:00] INFO: Processing package[apache2] action upgrade (ceph-qa::radosgw line 58) 2014-07-01 16:09:25,120.120 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:24-07:00] INFO: Processing package[libapache2-mod-fastcgi] action upgrade (ceph-qa::radosgw line 61) 2014-07-01 16:09:25,195.195 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libfcgi0ldbl] action install (ceph-qa::radosgw line 64) 2014-07-01 16:09:25,232.232 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: execute[apt-get update] ran successfully 2014-07-01 16:09:25,232.232 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[apache2] action upgrade (ceph-qa::radosgw line 58) 2014-07-01 16:09:25,277.277 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action disable (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,297.297 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libapache2-mod-fastcgi] action upgrade (ceph-qa::radosgw line 61) 2014-07-01 16:09:25,366.366 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action stop (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,373.373 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libfcgi0ldbl] action install (ceph-qa::radosgw line 64) 2014-07-01 16:09:25,425.425 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action disable (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,455.455 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[lsb-release] action install (ceph-qa::ubuntu line 176) 2014-07-01 16:09:25,506.506 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: execute[apt-get update] ran successfully 2014-07-01 16:09:25,507.507 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[apache2] action upgrade (ceph-qa::radosgw line 58) 2014-07-01 16:09:25,521.521 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action stop (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,534.534 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[build-essential] action install (ceph-qa::ubuntu line 177) 2014-07-01 16:09:25,582.582 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[lsb-release] action install (ceph-qa::ubuntu line 176) 2014-07-01 16:09:25,594.594 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libapache2-mod-fastcgi] action upgrade (ceph-qa::radosgw line 61) 2014-07-01 16:09:25,609.609 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[sysstat] action install (ceph-qa::ubuntu line 178) 2014-07-01 16:09:25,634.634 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[build-essential] action install (ceph-qa::ubuntu line 177) 2014-07-01 16:09:25,668.668 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libfcgi0ldbl] action install (ceph-qa::radosgw line 64) 2014-07-01 16:09:25,697.697 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[gdb] action install (ceph-qa::ubuntu line 179) 2014-07-01 16:09:25,714.714 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[sysstat] action install (ceph-qa::ubuntu line 178) 2014-07-01 16:09:25,743.743 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action disable (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,780.780 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[python-configobj] action install (ceph-qa::ubuntu line 180) 2014-07-01 16:09:25,784.784 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[gdb] action install (ceph-qa::ubuntu line 179) 2014-07-01 16:09:25,833.833 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing service[apache2] action stop (ceph-qa::radosgw line 66) 2014-07-01 16:09:25,844.844 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[python-configobj] action install (ceph-qa::ubuntu line 180) 2014-07-01 16:09:25,865.865 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[python-gevent] action install (ceph-qa::ubuntu line 181) 2014-07-01 16:09:25,907.907 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[python-gevent] action install (ceph-qa::ubuntu line 181) 2014-07-01 16:09:25,919.919 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[lsb-release] action install (ceph-qa::ubuntu line 176) 2014-07-01 16:09:25,930.930 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libedit2] action install (ceph-qa::ubuntu line 184) 2014-07-01 16:09:25,970.970 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libedit2] action install (ceph-qa::ubuntu line 184) 2014-07-01 16:09:25,993.993 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[build-essential] action install (ceph-qa::ubuntu line 177) 2014-07-01 16:09:26,012.012 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libssl0.9.8] action install (ceph-qa::ubuntu line 185) 2014-07-01 16:09:26,012.012 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libssl0.9.8] action install (ceph-qa::ubuntu line 185) 2014-07-01 16:09:26,054.054 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[sysstat] action install (ceph-qa::ubuntu line 178) 2014-07-01 16:09:26,090.090 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libgoogle-perftools0] action install (ceph-qa::ubuntu line 189) 2014-07-01 16:09:26,093.093 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libgoogle-perftools0] action install (ceph-qa::ubuntu line 189) 2014-07-01 16:09:26,138.138 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[gdb] action install (ceph-qa::ubuntu line 179) 2014-07-01 16:09:26,139.139 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:25-07:00] INFO: Processing package[libboost-thread1.46.1] action install (ceph-qa::ubuntu line 199) 2014-07-01 16:09:26,179.179 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libboost-thread1.46.1] action install (ceph-qa::ubuntu line 199) 2014-07-01 16:09:26,214.214 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[cryptsetup-bin] action install (ceph-qa::ubuntu line 203) 2014-07-01 16:09:26,226.226 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[python-configobj] action install (ceph-qa::ubuntu line 180) 2014-07-01 16:09:26,250.250 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[cryptsetup-bin] action install (ceph-qa::ubuntu line 203) 2014-07-01 16:09:26,268.268 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[xfsprogs] action install (ceph-qa::ubuntu line 204) 2014-07-01 16:09:26,299.299 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[python-gevent] action install (ceph-qa::ubuntu line 181) 2014-07-01 16:09:26,312.312 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[xfsprogs] action install (ceph-qa::ubuntu line 204) 2014-07-01 16:09:26,345.345 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[gdisk] action install (ceph-qa::ubuntu line 205) 2014-07-01 16:09:26,353.353 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libedit2] action install (ceph-qa::ubuntu line 184) 2014-07-01 16:09:26,393.393 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[parted] action install (ceph-qa::ubuntu line 206) 2014-07-01 16:09:26,393.393 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[gdisk] action install (ceph-qa::ubuntu line 205) 2014-07-01 16:09:26,418.418 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libssl0.9.8] action install (ceph-qa::ubuntu line 185) 2014-07-01 16:09:26,435.435 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[smbios-utils] action install (ceph-qa::ubuntu line 210) 2014-07-01 16:09:26,479.479 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libgoogle-perftools0] action install (ceph-qa::ubuntu line 189) 2014-07-01 16:09:26,479.479 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[parted] action install (ceph-qa::ubuntu line 206) 2014-07-01 16:09:26,480.480 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libcrypto++9] action install (ceph-qa::ubuntu line 219) 2014-07-01 16:09:26,480.480 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libboost-thread1.46.1] action install (ceph-qa::ubuntu line 199) 2014-07-01 16:09:26,494.494 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[smbios-utils] action install (ceph-qa::ubuntu line 210) 2014-07-01 16:09:26,521.521 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[openmpi-common] action remove (ceph-qa::ubuntu line 230) 2014-07-01 16:09:26,521.521 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[cryptsetup-bin] action install (ceph-qa::ubuntu line 203) 2014-07-01 16:09:26,552.552 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libcrypto++9] action install (ceph-qa::ubuntu line 219) 2014-07-01 16:09:26,585.585 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libuuid1] action install (ceph-qa::ubuntu line 234) 2014-07-01 16:09:26,591.591 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[xfsprogs] action install (ceph-qa::ubuntu line 204) 2014-07-01 16:09:26,612.612 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[openmpi-common] action remove (ceph-qa::ubuntu line 230) 2014-07-01 16:09:26,675.675 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[gdisk] action install (ceph-qa::ubuntu line 205) 2014-07-01 16:09:26,685.685 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libuuid1] action install (ceph-qa::ubuntu line 234) 2014-07-01 16:09:26,709.709 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libfcgi] action install (ceph-qa::ubuntu line 235) 2014-07-01 16:09:26,746.746 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[parted] action install (ceph-qa::ubuntu line 206) 2014-07-01 16:09:26,749.749 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libfcgi] action install (ceph-qa::ubuntu line 235) 2014-07-01 16:09:26,809.809 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[smbios-utils] action install (ceph-qa::ubuntu line 210) 2014-07-01 16:09:26,847.847 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: package[libfcgi] is a virtual package, actually acting on package[libfcgi0ldbl] 2014-07-01 16:09:26,867.867 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libcrypto++9] action install (ceph-qa::ubuntu line 219) 2014-07-01 16:09:26,878.878 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: package[libfcgi] is a virtual package, actually acting on package[libfcgi0ldbl] 2014-07-01 16:09:26,898.898 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[btrfs-tools] action install (ceph-qa::ubuntu line 236) 2014-07-01 16:09:26,927.927 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[openmpi-common] action remove (ceph-qa::ubuntu line 230) 2014-07-01 16:09:26,953.953 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libatomic-ops-dev] action install (ceph-qa::ubuntu line 239) 2014-07-01 16:09:26,957.957 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[btrfs-tools] action install (ceph-qa::ubuntu line 236) 2014-07-01 16:09:26,984.984 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libuuid1] action install (ceph-qa::ubuntu line 234) 2014-07-01 16:09:27,010.010 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[git-core] action install (ceph-qa::ubuntu line 242) 2014-07-01 16:09:27,039.039 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libfcgi] action install (ceph-qa::ubuntu line 235) 2014-07-01 16:09:27,042.042 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[libatomic-ops-dev] action install (ceph-qa::ubuntu line 239) 2014-07-01 16:09:27,065.065 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[attr] action install (ceph-qa::ubuntu line 243) 2014-07-01 16:09:27,122.122 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[dbench] action install (ceph-qa::ubuntu line 244) 2014-07-01 16:09:27,132.132 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:26-07:00] INFO: Processing package[git-core] action install (ceph-qa::ubuntu line 242) 2014-07-01 16:09:27,141.141 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: package[libfcgi] is a virtual package, actually acting on package[libfcgi0ldbl] 2014-07-01 16:09:27,180.180 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[bonnie++] action install (ceph-qa::ubuntu line 245) 2014-07-01 16:09:27,193.193 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[btrfs-tools] action install (ceph-qa::ubuntu line 236) 2014-07-01 16:09:27,209.209 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[attr] action install (ceph-qa::ubuntu line 243) 2014-07-01 16:09:27,235.235 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[iozone3] action install (ceph-qa::ubuntu line 246) 2014-07-01 16:09:27,243.243 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libatomic-ops-dev] action install (ceph-qa::ubuntu line 239) 2014-07-01 16:09:27,285.285 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[dbench] action install (ceph-qa::ubuntu line 244) 2014-07-01 16:09:27,296.296 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[tiobench] action install (ceph-qa::ubuntu line 248) 2014-07-01 16:09:27,296.296 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[git-core] action install (ceph-qa::ubuntu line 242) 2014-07-01 16:09:27,354.354 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[attr] action install (ceph-qa::ubuntu line 243) 2014-07-01 16:09:27,361.361 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[ltp-kernel-test] action install (ceph-qa::ubuntu line 263) 2014-07-01 16:09:27,390.390 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[bonnie++] action install (ceph-qa::ubuntu line 245) 2014-07-01 16:09:27,410.410 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[dbench] action install (ceph-qa::ubuntu line 244) 2014-07-01 16:09:27,416.416 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[valgrind] action install (ceph-qa::ubuntu line 265) 2014-07-01 16:09:27,472.472 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[bonnie++] action install (ceph-qa::ubuntu line 245) 2014-07-01 16:09:27,484.484 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[python-nose] action install (ceph-qa::ubuntu line 266) 2014-07-01 16:09:27,491.491 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[iozone3] action install (ceph-qa::ubuntu line 246) 2014-07-01 16:09:27,522.522 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[iozone3] action install (ceph-qa::ubuntu line 246) 2014-07-01 16:09:27,540.540 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[mpich2] action install (ceph-qa::ubuntu line 267) 2014-07-01 16:09:27,555.555 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[tiobench] action install (ceph-qa::ubuntu line 248) 2014-07-01 16:09:27,570.570 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[tiobench] action install (ceph-qa::ubuntu line 248) 2014-07-01 16:09:27,571.571 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-3] action install (ceph-qa::ubuntu line 269) 2014-07-01 16:09:27,609.609 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[ltp-kernel-test] action install (ceph-qa::ubuntu line 263) 2014-07-01 16:09:27,619.619 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-dev] action install (ceph-qa::ubuntu line 273) 2014-07-01 16:09:27,655.655 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[ltp-kernel-test] action install (ceph-qa::ubuntu line 263) 2014-07-01 16:09:27,665.665 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[valgrind] action install (ceph-qa::ubuntu line 265) 2014-07-01 16:09:27,672.672 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[ant] action install (ceph-qa::ubuntu line 274) 2014-07-01 16:09:27,715.715 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[python-nose] action install (ceph-qa::ubuntu line 266) 2014-07-01 16:09:27,723.723 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libtool] action install (ceph-qa::ubuntu line 277) 2014-07-01 16:09:27,738.738 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[valgrind] action install (ceph-qa::ubuntu line 265) 2014-07-01 16:09:27,770.770 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[mpich2] action install (ceph-qa::ubuntu line 267) 2014-07-01 16:09:27,784.784 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[automake] action install (ceph-qa::ubuntu line 278) 2014-07-01 16:09:27,825.825 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-3] action install (ceph-qa::ubuntu line 269) 2014-07-01 16:09:27,825.825 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[python-nose] action install (ceph-qa::ubuntu line 266) 2014-07-01 16:09:27,826.826 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[gettext] action install (ceph-qa::ubuntu line 279) 2014-07-01 16:09:27,874.874 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[uuid-dev] action install (ceph-qa::ubuntu line 280) 2014-07-01 16:09:27,877.877 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-dev] action install (ceph-qa::ubuntu line 273) 2014-07-01 16:09:27,923.923 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[mpich2] action install (ceph-qa::ubuntu line 267) 2014-07-01 16:09:27,929.929 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[ant] action install (ceph-qa::ubuntu line 274) 2014-07-01 16:09:27,932.932 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libacl1-dev] action install (ceph-qa::ubuntu line 281) 2014-07-01 16:09:27,978.978 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libtool] action install (ceph-qa::ubuntu line 277) 2014-07-01 16:09:27,984.984 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[bc] action install (ceph-qa::ubuntu line 282) 2014-07-01 16:09:28,017.017 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-3] action install (ceph-qa::ubuntu line 269) 2014-07-01 16:09:28,028.028 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[automake] action install (ceph-qa::ubuntu line 278) 2014-07-01 16:09:28,036.036 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[xfsdump] action install (ceph-qa::ubuntu line 283) 2014-07-01 16:09:28,078.078 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[gettext] action install (ceph-qa::ubuntu line 279) 2014-07-01 16:09:28,097.097 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[dmapi] action install (ceph-qa::ubuntu line 284) 2014-07-01 16:09:28,115.115 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[libmpich2-dev] action install (ceph-qa::ubuntu line 273) 2014-07-01 16:09:28,135.135 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:27-07:00] INFO: Processing package[uuid-dev] action install (ceph-qa::ubuntu line 280) 2014-07-01 16:09:28,177.177 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: package[dmapi] is a virtual package, actually acting on package[libdm0] 2014-07-01 16:09:28,193.193 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libacl1-dev] action install (ceph-qa::ubuntu line 281) 2014-07-01 16:09:28,204.204 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[ant] action install (ceph-qa::ubuntu line 274) 2014-07-01 16:09:28,229.229 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xfslibs-dev] action install (ceph-qa::ubuntu line 285) 2014-07-01 16:09:28,245.245 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[bc] action install (ceph-qa::ubuntu line 282) 2014-07-01 16:09:28,279.279 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libattr1-dev] action install (ceph-qa::ubuntu line 286) 2014-07-01 16:09:28,301.301 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libtool] action install (ceph-qa::ubuntu line 277) 2014-07-01 16:09:28,307.307 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xfsdump] action install (ceph-qa::ubuntu line 283) 2014-07-01 16:09:28,336.336 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[sysprof] action install (ceph-qa::ubuntu line 290) 2014-07-01 16:09:28,352.352 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[dmapi] action install (ceph-qa::ubuntu line 284) 2014-07-01 16:09:28,391.391 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[automake] action install (ceph-qa::ubuntu line 278) 2014-07-01 16:09:28,400.400 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[vim] action install (ceph-qa::ubuntu line 292) 2014-07-01 16:09:28,440.440 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: package[dmapi] is a virtual package, actually acting on package[libdm0] 2014-07-01 16:09:28,452.452 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[pdsh] action install (ceph-qa::ubuntu line 293) 2014-07-01 16:09:28,491.491 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[gettext] action install (ceph-qa::ubuntu line 279) 2014-07-01 16:09:28,499.499 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[collectl] action install (ceph-qa::ubuntu line 294) 2014-07-01 16:09:28,523.523 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xfslibs-dev] action install (ceph-qa::ubuntu line 285) 2014-07-01 16:09:28,556.556 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing service[collectl] action disable (ceph-qa::ubuntu line 295) 2014-07-01 16:09:28,582.582 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[uuid-dev] action install (ceph-qa::ubuntu line 280) 2014-07-01 16:09:28,604.604 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libattr1-dev] action install (ceph-qa::ubuntu line 286) 2014-07-01 16:09:28,614.614 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing service[collectl] action stop (ceph-qa::ubuntu line 295) 2014-07-01 16:09:28,668.668 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libacl1-dev] action install (ceph-qa::ubuntu line 281) 2014-07-01 16:09:28,679.679 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[sysprof] action install (ceph-qa::ubuntu line 290) 2014-07-01 16:09:28,679.679 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[blktrace] action install (ceph-qa::ubuntu line 300) 2014-07-01 16:09:28,737.737 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[python-numpy] action install (ceph-qa::ubuntu line 301) 2014-07-01 16:09:28,751.751 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[vim] action install (ceph-qa::ubuntu line 292) 2014-07-01 16:09:28,785.785 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[bc] action install (ceph-qa::ubuntu line 282) 2014-07-01 16:09:28,788.788 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[python-matplotlib] action install (ceph-qa::ubuntu line 302) 2014-07-01 16:09:28,834.834 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[pdsh] action install (ceph-qa::ubuntu line 293) 2014-07-01 16:09:28,836.836 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[mencoder] action install (ceph-qa::ubuntu line 303) 2014-07-01 16:09:28,854.854 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xfsdump] action install (ceph-qa::ubuntu line 283) 2014-07-01 16:09:28,889.889 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[kvm] action install (ceph-qa::ubuntu line 309) 2014-07-01 16:09:28,918.918 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[collectl] action install (ceph-qa::ubuntu line 294) 2014-07-01 16:09:28,920.920 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[dmapi] action install (ceph-qa::ubuntu line 284) 2014-07-01 16:09:28,944.944 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[genisoimage] action install (ceph-qa::ubuntu line 311) 2014-07-01 16:09:28,989.989 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[libjson-xs-perl] action install (ceph-qa::ubuntu line 314) 2014-07-01 16:09:29,002.002 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing service[collectl] action disable (ceph-qa::ubuntu line 295) 2014-07-01 16:09:29,041.041 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: package[dmapi] is a virtual package, actually acting on package[libdm0] 2014-07-01 16:09:29,046.046 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xml-twig-tools] action install (ceph-qa::ubuntu line 316) 2014-07-01 16:09:29,086.086 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing service[collectl] action stop (ceph-qa::ubuntu line 295) 2014-07-01 16:09:29,106.106 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[default-jdk] action install (ceph-qa::ubuntu line 319) 2014-07-01 16:09:29,109.109 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:28-07:00] INFO: Processing package[xfslibs-dev] action install (ceph-qa::ubuntu line 285) 2014-07-01 16:09:29,167.167 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[libattr1-dev] action install (ceph-qa::ubuntu line 286) 2014-07-01 16:09:29,173.173 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[blktrace] action install (ceph-qa::ubuntu line 300) 2014-07-01 16:09:29,178.178 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[junit4] action install (ceph-qa::ubuntu line 320) 2014-07-01 16:09:29,217.217 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[sysprof] action install (ceph-qa::ubuntu line 290) 2014-07-01 16:09:29,251.251 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[python-numpy] action install (ceph-qa::ubuntu line 301) 2014-07-01 16:09:29,255.255 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[tgt] action install (ceph-qa::ubuntu line 323) 2014-07-01 16:09:29,281.281 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[vim] action install (ceph-qa::ubuntu line 292) 2014-07-01 16:09:29,332.332 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[python-matplotlib] action install (ceph-qa::ubuntu line 302) 2014-07-01 16:09:29,334.334 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[open-iscsi] action install (ceph-qa::ubuntu line 326) 2014-07-01 16:09:29,344.344 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[pdsh] action install (ceph-qa::ubuntu line 293) 2014-07-01 16:09:29,410.410 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[mencoder] action install (ceph-qa::ubuntu line 303) 2014-07-01 16:09:29,413.413 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[smartmontools] action install (ceph-qa::ubuntu line 328) 2014-07-01 16:09:29,417.417 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[collectl] action install (ceph-qa::ubuntu line 294) 2014-07-01 16:09:29,489.489 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing service[collectl] action disable (ceph-qa::ubuntu line 295) 2014-07-01 16:09:29,492.492 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[kvm] action install (ceph-qa::ubuntu line 309) 2014-07-01 16:09:29,495.495 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[nagios-nrpe-server] action install (ceph-qa::ubuntu line 329) 2014-07-01 16:09:29,553.553 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing service[collectl] action stop (ceph-qa::ubuntu line 295) 2014-07-01 16:09:29,563.563 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[genisoimage] action install (ceph-qa::ubuntu line 311) 2014-07-01 16:09:29,577.577 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[cifs-utils] action install (ceph-qa::ubuntu line 332) 2014-07-01 16:09:29,622.622 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[blktrace] action install (ceph-qa::ubuntu line 300) 2014-07-01 16:09:29,651.651 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[libjson-xs-perl] action install (ceph-qa::ubuntu line 314) 2014-07-01 16:09:29,654.654 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[distcc] action install (ceph-qa::ubuntu line 335) 2014-07-01 16:09:29,691.691 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[python-numpy] action install (ceph-qa::ubuntu line 301) 2014-07-01 16:09:29,724.724 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[xml-twig-tools] action install (ceph-qa::ubuntu line 316) 2014-07-01 16:09:29,731.731 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing file[/etc/grub.d/02_force_timeout] action create (ceph-qa::ubuntu line 351) 2014-07-01 16:09:29,748.748 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[ntp] action install (ceph-qa::ubuntu line 364) 2014-07-01 16:09:29,748.748 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[python-matplotlib] action install (ceph-qa::ubuntu line 302) 2014-07-01 16:09:29,806.806 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[default-jdk] action install (ceph-qa::ubuntu line 319) 2014-07-01 16:09:29,807.807 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[mencoder] action install (ceph-qa::ubuntu line 303) 2014-07-01 16:09:29,830.830 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing cookbook_file[/etc/ntp.conf] action create (ceph-qa::ubuntu line 366) 2014-07-01 16:09:29,830.830 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing service[ntp] action enable (ceph-qa::ubuntu line 374) 2014-07-01 16:09:29,860.860 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[kvm] action install (ceph-qa::ubuntu line 309) 2014-07-01 16:09:29,891.891 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[junit4] action install (ceph-qa::ubuntu line 320) 2014-07-01 16:09:29,921.921 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing service[ntp] action start (ceph-qa::ubuntu line 374) 2014-07-01 16:09:29,924.924 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[genisoimage] action install (ceph-qa::ubuntu line 311) 2014-07-01 16:09:29,975.975 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[tgt] action install (ceph-qa::ubuntu line 323) 2014-07-01 16:09:29,993.993 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[libjson-xs-perl] action install (ceph-qa::ubuntu line 314) 2014-07-01 16:09:30,009.009 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing execute[add user_xattr to root mount options in fstab] action run (ceph-qa::ubuntu line 378) 2014-07-01 16:09:30,054.054 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[open-iscsi] action install (ceph-qa::ubuntu line 326) 2014-07-01 16:09:30,055.055 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[xml-twig-tools] action install (ceph-qa::ubuntu line 316) 2014-07-01 16:09:30,060.060 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: execute[add user_xattr to root mount options in fstab] ran successfully 2014-07-01 16:09:30,060.060 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing execute[enable xattr for this boot] action run (ceph-qa::ubuntu line 387) 2014-07-01 16:09:30,102.102 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[default-jdk] action install (ceph-qa::ubuntu line 319) 2014-07-01 16:09:30,129.129 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:29-07:00] INFO: Processing package[smartmontools] action install (ceph-qa::ubuntu line 328) 2014-07-01 16:09:30,145.145 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[enable xattr for this boot] ran successfully 2014-07-01 16:09:30,145.145 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[allow fuse mounts to be used by non-owners] action run (ceph-qa::ubuntu line 391) 2014-07-01 16:09:30,169.169 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[junit4] action install (ceph-qa::ubuntu line 320) 2014-07-01 16:09:30,206.206 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[nagios-nrpe-server] action install (ceph-qa::ubuntu line 329) 2014-07-01 16:09:30,225.225 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[allow fuse mounts to be used by non-owners] ran successfully 2014-07-01 16:09:30,225.225 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[add user ubuntu to group fuse] action run (ceph-qa::ubuntu line 395) 2014-07-01 16:09:30,229.229 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[tgt] action install (ceph-qa::ubuntu line 323) 2014-07-01 16:09:30,296.296 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[open-iscsi] action install (ceph-qa::ubuntu line 326) 2014-07-01 16:09:30,309.309 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[cifs-utils] action install (ceph-qa::ubuntu line 332) 2014-07-01 16:09:30,359.359 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[smartmontools] action install (ceph-qa::ubuntu line 328) 2014-07-01 16:09:30,370.370 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[distcc] action install (ceph-qa::ubuntu line 335) 2014-07-01 16:09:30,422.422 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[nagios-nrpe-server] action install (ceph-qa::ubuntu line 329) 2014-07-01 16:09:30,432.432 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing file[/etc/grub.d/02_force_timeout] action create (ceph-qa::ubuntu line 351) 2014-07-01 16:09:30,436.436 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[ntp] action install (ceph-qa::ubuntu line 364) 2014-07-01 16:09:30,442.442 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[add user ubuntu to group fuse] ran successfully 2014-07-01 16:09:30,442.442 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing file[/etc/fuse.conf] action create (ceph-qa::ubuntu line 399) 2014-07-01 16:09:30,442.442 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[add user ubuntu to group kvm] action run (ceph-qa::ubuntu line 403) 2014-07-01 16:09:30,482.482 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[cifs-utils] action install (ceph-qa::ubuntu line 332) 2014-07-01 16:09:30,487.487 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing cookbook_file[/etc/ntp.conf] action create (ceph-qa::ubuntu line 366) 2014-07-01 16:09:30,502.502 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing service[ntp] action enable (ceph-qa::ubuntu line 374) 2014-07-01 16:09:30,535.535 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[add user ubuntu to group kvm] ran successfully 2014-07-01 16:09:30,537.537 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing directory[/home/ubuntu/.ssh] action create (ceph-qa::ubuntu line 407) 2014-07-01 16:09:30,541.541 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[set up ssh keys] action run (ceph-qa::ubuntu line 413) 2014-07-01 16:09:30,555.555 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[distcc] action install (ceph-qa::ubuntu line 335) 2014-07-01 16:09:30,567.567 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing service[ntp] action start (ceph-qa::ubuntu line 374) 2014-07-01 16:09:30,623.623 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing file[/etc/grub.d/02_force_timeout] action create (ceph-qa::ubuntu line 351) 2014-07-01 16:09:30,626.626 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing package[ntp] action install (ceph-qa::ubuntu line 364) 2014-07-01 16:09:30,643.643 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[add user_xattr to root mount options in fstab] action run (ceph-qa::ubuntu line 378) 2014-07-01 16:09:30,681.681 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing cookbook_file[/etc/ntp.conf] action create (ceph-qa::ubuntu line 366) 2014-07-01 16:09:30,685.685 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing service[ntp] action enable (ceph-qa::ubuntu line 374) 2014-07-01 16:09:30,686.686 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[add user_xattr to root mount options in fstab] ran successfully 2014-07-01 16:09:30,689.689 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[enable xattr for this boot] action run (ceph-qa::ubuntu line 387) 2014-07-01 16:09:30,756.756 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing service[ntp] action start (ceph-qa::ubuntu line 374) 2014-07-01 16:09:30,819.819 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[add user_xattr to root mount options in fstab] action run (ceph-qa::ubuntu line 378) 2014-07-01 16:09:30,861.861 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[add user_xattr to root mount options in fstab] ran successfully 2014-07-01 16:09:30,864.864 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[enable xattr for this boot] action run (ceph-qa::ubuntu line 387) 2014-07-01 16:09:30,947.947 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[enable xattr for this boot] ran successfully 2014-07-01 16:09:30,967.967 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[allow fuse mounts to be used by non-owners] action run (ceph-qa::ubuntu line 391) 2014-07-01 16:09:31,028.028 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[allow fuse mounts to be used by non-owners] ran successfully 2014-07-01 16:09:31,028.028 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[add user ubuntu to group fuse] action run (ceph-qa::ubuntu line 395) 2014-07-01 16:09:31,086.086 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[enable xattr for this boot] ran successfully 2014-07-01 16:09:31,090.090 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing execute[allow fuse mounts to be used by non-owners] action run (ceph-qa::ubuntu line 391) 2014-07-01 16:09:31,093.093 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: execute[set up ssh keys] ran successfully 2014-07-01 16:09:31,094.094 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:30-07:00] INFO: Processing cookbook_file[/usr/sbin/megacli] action create (ceph-qa::ubuntu line 425) 2014-07-01 16:09:31,158.158 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[allow fuse mounts to be used by non-owners] ran successfully 2014-07-01 16:09:31,158.158 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add user ubuntu to group fuse] action run (ceph-qa::ubuntu line 395) 2014-07-01 16:09:31,244.244 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add user ubuntu to group fuse] ran successfully 2014-07-01 16:09:31,245.245 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing file[/etc/fuse.conf] action create (ceph-qa::ubuntu line 399) 2014-07-01 16:09:31,245.245 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add user ubuntu to group kvm] action run (ceph-qa::ubuntu line 403) 2014-07-01 16:09:31,277.277 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/sbin/cli64] action create (ceph-qa::ubuntu line 431) 2014-07-01 16:09:31,370.370 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing directory[/usr/libexec/] action create (ceph-qa::ubuntu line 440) 2014-07-01 16:09:31,371.371 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/raid.pl] action create (ceph-qa::ubuntu line 446) 2014-07-01 16:09:31,372.372 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add user ubuntu to group kvm] ran successfully 2014-07-01 16:09:31,373.373 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/smart.pl] action create (ceph-qa::ubuntu line 452) 2014-07-01 16:09:31,373.373 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing directory[/home/ubuntu/.ssh] action create (ceph-qa::ubuntu line 407) 2014-07-01 16:09:31,374.374 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/diskusage.pl] action create (ceph-qa::ubuntu line 458) 2014-07-01 16:09:31,374.374 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/etc/ssh/ssh_config] action create (ceph-qa::ubuntu line 467) 2014-07-01 16:09:31,375.375 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[set up ssh keys] action run (ceph-qa::ubuntu line 413) 2014-07-01 16:09:31,376.376 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add ubuntu to disk group] action run (ceph-qa::ubuntu line 474) 2014-07-01 16:09:31,395.395 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add user ubuntu to group fuse] ran successfully 2014-07-01 16:09:31,396.396 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing file[/etc/fuse.conf] action create (ceph-qa::ubuntu line 399) 2014-07-01 16:09:31,399.399 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add user ubuntu to group kvm] action run (ceph-qa::ubuntu line 403) 2014-07-01 16:09:31,458.458 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add ubuntu to disk group] ran successfully 2014-07-01 16:09:31,458.458 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[enable kernel logging to console] action run (ceph-qa::ubuntu line 482) 2014-07-01 16:09:31,459.459 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add user ubuntu to group kvm] ran successfully 2014-07-01 16:09:31,460.460 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing directory[/home/ubuntu/.ssh] action create (ceph-qa::ubuntu line 407) 2014-07-01 16:09:31,468.468 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[set up ssh keys] action run (ceph-qa::ubuntu line 413) 2014-07-01 16:09:31,540.540 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[enable kernel logging to console] ran successfully 2014-07-01 16:09:31,541.541 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[update-grub] action run (ceph-qa::ubuntu line 539) 2014-07-01 16:09:31,628.628 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[set up ssh keys] ran successfully 2014-07-01 16:09:31,629.629 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/sbin/megacli] action create (ceph-qa::ubuntu line 425) 2014-07-01 16:09:31,674.674 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[set up ssh keys] ran successfully 2014-07-01 16:09:31,674.674 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/sbin/megacli] action create (ceph-qa::ubuntu line 425) 2014-07-01 16:09:31,716.716 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/sbin/cli64] action create (ceph-qa::ubuntu line 431) 2014-07-01 16:09:31,758.758 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing directory[/usr/libexec/] action create (ceph-qa::ubuntu line 440) 2014-07-01 16:09:31,758.758 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/raid.pl] action create (ceph-qa::ubuntu line 446) 2014-07-01 16:09:31,759.759 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/smart.pl] action create (ceph-qa::ubuntu line 452) 2014-07-01 16:09:31,760.760 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/diskusage.pl] action create (ceph-qa::ubuntu line 458) 2014-07-01 16:09:31,764.764 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/etc/ssh/ssh_config] action create (ceph-qa::ubuntu line 467) 2014-07-01 16:09:31,777.777 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add ubuntu to disk group] action run (ceph-qa::ubuntu line 474) 2014-07-01 16:09:31,795.795 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/sbin/cli64] action create (ceph-qa::ubuntu line 431) 2014-07-01 16:09:31,836.836 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing directory[/usr/libexec/] action create (ceph-qa::ubuntu line 440) 2014-07-01 16:09:31,842.842 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/raid.pl] action create (ceph-qa::ubuntu line 446) 2014-07-01 16:09:31,842.842 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/smart.pl] action create (ceph-qa::ubuntu line 452) 2014-07-01 16:09:31,843.843 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/usr/libexec/diskusage.pl] action create (ceph-qa::ubuntu line 458) 2014-07-01 16:09:31,843.843 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing cookbook_file[/etc/ssh/ssh_config] action create (ceph-qa::ubuntu line 467) 2014-07-01 16:09:31,846.846 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[add ubuntu to disk group] action run (ceph-qa::ubuntu line 474) 2014-07-01 16:09:31,875.875 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add ubuntu to disk group] ran successfully 2014-07-01 16:09:31,876.876 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[enable kernel logging to console] action run (ceph-qa::ubuntu line 482) 2014-07-01 16:09:31,917.917 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[add ubuntu to disk group] ran successfully 2014-07-01 16:09:31,917.917 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[enable kernel logging to console] action run (ceph-qa::ubuntu line 482) 2014-07-01 16:09:31,968.968 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[enable kernel logging to console] ran successfully 2014-07-01 16:09:31,969.969 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[update-grub] action run (ceph-qa::ubuntu line 539) 2014-07-01 16:09:31,990.990 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: execute[enable kernel logging to console] ran successfully 2014-07-01 16:09:31,992.992 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:31-07:00] INFO: Processing execute[update-grub] action run (ceph-qa::ubuntu line 539) 2014-07-01 16:09:35,562.562 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: execute[update-grub] ran successfully 2014-07-01 16:09:35,566.566 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing cookbook_file[/etc/init/ttyS1.conf] action create (ceph-qa::ubuntu line 545) 2014-07-01 16:09:35,566.566 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing cookbook_file[/etc/init/ttyS2.conf] action create (ceph-qa::ubuntu line 554) 2014-07-01 16:09:35,569.569 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing service[ttyS1] action enable (ceph-qa::ubuntu line 563) 2014-07-01 16:09:35,582.582 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing service[ttyS1] action start (ceph-qa::ubuntu line 563) 2014-07-01 16:09:35,592.592 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing service[ttyS2] action enable (ceph-qa::ubuntu line 571) 2014-07-01 16:09:35,599.599 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing service[ttyS2] action start (ceph-qa::ubuntu line 571) 2014-07-01 16:09:35,610.610 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing package[nfs-kernel-server] action install (ceph-qa::ubuntu line 582) 2014-07-01 16:09:35,668.668 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing package[ipcalc] action install (ceph-qa::ubuntu line 585) 2014-07-01 16:09:35,721.721 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing execute[set up static IP in /etc/hosts] action run (ceph-qa::ubuntu line 589) 2014-07-01 16:09:35,779.779 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: execute[set up static IP in /etc/hosts] ran successfully 2014-07-01 16:09:35,779.779 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing execute[set up static IP and 10gig interface] action run (ceph-qa::ubuntu line 597) 2014-07-01 16:09:35,797.797 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: execute[set up static IP and 10gig interface] ran successfully 2014-07-01 16:09:35,797.797 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:35-07:00] INFO: Processing execute[Installing CPAN Amazon::S3] action run (ceph-qa::ubuntu line 649) 2014-07-01 16:09:36,684.684 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: execute[update-grub] ran successfully 2014-07-01 16:09:36,685.685 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing cookbook_file[/etc/init/ttyS1.conf] action create (ceph-qa::ubuntu line 545) 2014-07-01 16:09:36,685.685 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing cookbook_file[/etc/init/ttyS2.conf] action create (ceph-qa::ubuntu line 554) 2014-07-01 16:09:36,690.690 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing service[ttyS1] action enable (ceph-qa::ubuntu line 563) 2014-07-01 16:09:36,695.695 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing service[ttyS1] action start (ceph-qa::ubuntu line 563) 2014-07-01 16:09:36,697.697 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing service[ttyS2] action enable (ceph-qa::ubuntu line 571) 2014-07-01 16:09:36,719.719 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing service[ttyS2] action start (ceph-qa::ubuntu line 571) 2014-07-01 16:09:36,729.729 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing package[nfs-kernel-server] action install (ceph-qa::ubuntu line 582) 2014-07-01 16:09:36,815.815 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing package[ipcalc] action install (ceph-qa::ubuntu line 585) 2014-07-01 16:09:36,895.895 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing execute[set up static IP in /etc/hosts] action run (ceph-qa::ubuntu line 589) 2014-07-01 16:09:36,947.947 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: execute[set up static IP in /etc/hosts] ran successfully 2014-07-01 16:09:36,947.947 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing execute[set up static IP and 10gig interface] action run (ceph-qa::ubuntu line 597) 2014-07-01 16:09:37,025.025 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: execute[set up static IP and 10gig interface] ran successfully 2014-07-01 16:09:37,025.025 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:36-07:00] INFO: Processing execute[Installing CPAN Amazon::S3] action run (ceph-qa::ubuntu line 649) 2014-07-01 16:09:37,498.498 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: execute[update-grub] ran successfully 2014-07-01 16:09:37,500.500 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing cookbook_file[/etc/init/ttyS1.conf] action create (ceph-qa::ubuntu line 545) 2014-07-01 16:09:37,500.500 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing cookbook_file[/etc/init/ttyS2.conf] action create (ceph-qa::ubuntu line 554) 2014-07-01 16:09:37,500.500 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing service[ttyS1] action enable (ceph-qa::ubuntu line 563) 2014-07-01 16:09:37,501.501 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing service[ttyS1] action start (ceph-qa::ubuntu line 563) 2014-07-01 16:09:37,501.501 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing service[ttyS2] action enable (ceph-qa::ubuntu line 571) 2014-07-01 16:09:37,502.502 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing service[ttyS2] action start (ceph-qa::ubuntu line 571) 2014-07-01 16:09:37,503.503 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing package[nfs-kernel-server] action install (ceph-qa::ubuntu line 582) 2014-07-01 16:09:37,571.571 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing package[ipcalc] action install (ceph-qa::ubuntu line 585) 2014-07-01 16:09:37,659.659 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing execute[set up static IP in /etc/hosts] action run (ceph-qa::ubuntu line 589) 2014-07-01 16:09:37,714.714 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: execute[set up static IP in /etc/hosts] ran successfully 2014-07-01 16:09:37,714.714 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing execute[set up static IP and 10gig interface] action run (ceph-qa::ubuntu line 597) 2014-07-01 16:09:37,761.761 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: execute[set up static IP and 10gig interface] ran successfully 2014-07-01 16:09:37,761.761 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing execute[Installing CPAN Amazon::S3] action run (ceph-qa::ubuntu line 649) 2014-07-01 16:09:38,124.124 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: execute[Installing CPAN Amazon::S3] ran successfully 2014-07-01 16:09:38,124.124 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing file[/etc/resolvconf/resolv.conf.d/base] action create (ceph-qa::ubuntu line 654) 2014-07-01 16:09:38,125.125 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing file[/etc/sudoers.d/90-nagios] action create (ceph-qa::ubuntu line 666) 2014-07-01 16:09:38,128.128 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing cookbook_file[/etc/modules] action create (ceph-qa::ubuntu line 676) 2014-07-01 16:09:38,132.132 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing cookbook_file[/etc/nagios/nrpe.cfg] action create (ceph-qa::ubuntu line 684) 2014-07-01 16:09:38,135.135 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:37-07:00] INFO: Processing service[nagios-nrpe-server] action enable (ceph-qa::ubuntu line 692) 2014-07-01 16:09:38,199.199 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing service[nagios-nrpe-server] action start (ceph-qa::ubuntu line 692) 2014-07-01 16:09:38,284.284 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: service[nagios-nrpe-server] started 2014-07-01 16:09:38,285.285 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing file[/etc/default/nagios-nrpe-server] action create (ceph-qa::ubuntu line 697) 2014-07-01 16:09:38,288.288 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing execute[Restarting resolvdns] action run (ceph-qa::ubuntu line 706) 2014-07-01 16:09:38,384.384 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: execute[Restarting resolvdns] ran successfully 2014-07-01 16:09:38,385.385 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing execute[Enabling auto-fsck fix to prevent boot hangup] action run (ceph-qa::ubuntu line 712) 2014-07-01 16:09:38,433.433 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: execute[Enabling auto-fsck fix to prevent boot hangup] ran successfully 2014-07-01 16:09:38,434.434 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing bash[ssh_max_sessions] action run (ceph-qa::ubuntu line 718) 2014-07-01 16:09:38,434.434 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:38-07:00] INFO: Processing service[ssh] action restart (ceph-qa::ubuntu line 734) 2014-07-01 16:09:39,318.318 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: execute[Installing CPAN Amazon::S3] ran successfully 2014-07-01 16:09:39,319.319 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing file[/etc/resolvconf/resolv.conf.d/base] action create (ceph-qa::ubuntu line 654) 2014-07-01 16:09:39,319.319 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing file[/etc/sudoers.d/90-nagios] action create (ceph-qa::ubuntu line 666) 2014-07-01 16:09:39,322.322 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing cookbook_file[/etc/modules] action create (ceph-qa::ubuntu line 676) 2014-07-01 16:09:39,325.325 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing cookbook_file[/etc/nagios/nrpe.cfg] action create (ceph-qa::ubuntu line 684) 2014-07-01 16:09:39,329.329 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing service[nagios-nrpe-server] action enable (ceph-qa::ubuntu line 692) 2014-07-01 16:09:39,405.405 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing service[nagios-nrpe-server] action start (ceph-qa::ubuntu line 692) 2014-07-01 16:09:39,537.537 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: service[nagios-nrpe-server] started 2014-07-01 16:09:39,537.537 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing file[/etc/default/nagios-nrpe-server] action create (ceph-qa::ubuntu line 697) 2014-07-01 16:09:39,541.541 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing execute[Restarting resolvdns] action run (ceph-qa::ubuntu line 706) 2014-07-01 16:09:39,657.657 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:39-07:00] INFO: service[ssh] restarted 2014-07-01 16:09:39,658.658 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing file[/ceph-qa-ready] action create (ceph-qa::ubuntu line 739) 2014-07-01 16:09:39,666.666 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: execute[Restarting resolvdns] ran successfully 2014-07-01 16:09:39,667.667 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing execute[Enabling auto-fsck fix to prevent boot hangup] action run (ceph-qa::ubuntu line 712) 2014-07-01 16:09:39,680.680 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:39-07:00] INFO: Chef Run complete in 28.902915694 seconds 2014-07-01 16:09:39,680.680 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:39-07:00] INFO: Running report handlers 2014-07-01 16:09:39,680.680 INFO:teuthology.orchestra.run.mira079.stdout:[2014-07-01T07:09:39-07:00] INFO: Report handlers complete 2014-07-01 16:09:39,738.738 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: execute[Enabling auto-fsck fix to prevent boot hangup] ran successfully 2014-07-01 16:09:39,738.738 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing bash[ssh_max_sessions] action run (ceph-qa::ubuntu line 718) 2014-07-01 16:09:39,739.739 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:39-07:00] INFO: Processing service[ssh] action restart (ceph-qa::ubuntu line 734) 2014-07-01 16:09:39,761.761 INFO:teuthology.orchestra.run.mira079.stderr:+ cleanup 2014-07-01 16:09:39,761.761 INFO:teuthology.orchestra.run.mira079.stderr:+ rm -rf /tmp/solo-from-scratch.nJK73qjNeLkS 2014-07-01 16:09:40,310.310 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: execute[Installing CPAN Amazon::S3] ran successfully 2014-07-01 16:09:40,310.310 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing file[/etc/resolvconf/resolv.conf.d/base] action create (ceph-qa::ubuntu line 654) 2014-07-01 16:09:40,312.312 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing file[/etc/sudoers.d/90-nagios] action create (ceph-qa::ubuntu line 666) 2014-07-01 16:09:40,313.313 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing cookbook_file[/etc/modules] action create (ceph-qa::ubuntu line 676) 2014-07-01 16:09:40,314.314 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing cookbook_file[/etc/nagios/nrpe.cfg] action create (ceph-qa::ubuntu line 684) 2014-07-01 16:09:40,314.314 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing service[nagios-nrpe-server] action enable (ceph-qa::ubuntu line 692) 2014-07-01 16:09:40,406.406 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing service[nagios-nrpe-server] action start (ceph-qa::ubuntu line 692) 2014-07-01 16:09:40,478.478 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: service[nagios-nrpe-server] started 2014-07-01 16:09:40,479.479 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing file[/etc/default/nagios-nrpe-server] action create (ceph-qa::ubuntu line 697) 2014-07-01 16:09:40,482.482 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing execute[Restarting resolvdns] action run (ceph-qa::ubuntu line 706) 2014-07-01 16:09:40,604.604 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: execute[Restarting resolvdns] ran successfully 2014-07-01 16:09:40,604.604 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing execute[Enabling auto-fsck fix to prevent boot hangup] action run (ceph-qa::ubuntu line 712) 2014-07-01 16:09:40,649.649 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: execute[Enabling auto-fsck fix to prevent boot hangup] ran successfully 2014-07-01 16:09:40,653.653 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing bash[ssh_max_sessions] action run (ceph-qa::ubuntu line 718) 2014-07-01 16:09:40,655.655 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing service[ssh] action restart (ceph-qa::ubuntu line 734) 2014-07-01 16:09:41,043.043 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:40-07:00] INFO: service[ssh] restarted 2014-07-01 16:09:41,043.043 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:40-07:00] INFO: Processing file[/ceph-qa-ready] action create (ceph-qa::ubuntu line 739) 2014-07-01 16:09:41,065.065 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:40-07:00] INFO: Chef Run complete in 30.51593541 seconds 2014-07-01 16:09:41,065.065 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:40-07:00] INFO: Running report handlers 2014-07-01 16:09:41,066.066 INFO:teuthology.orchestra.run.mira043.stdout:[2014-07-01T07:09:40-07:00] INFO: Report handlers complete 2014-07-01 16:09:41,150.150 INFO:teuthology.orchestra.run.mira043.stderr:+ cleanup 2014-07-01 16:09:41,151.151 INFO:teuthology.orchestra.run.mira043.stderr:+ rm -rf /tmp/solo-from-scratch.t9TrCQTgYTKi 2014-07-01 16:09:41,900.900 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:41-07:00] INFO: service[ssh] restarted 2014-07-01 16:09:41,901.901 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:41-07:00] INFO: Processing file[/ceph-qa-ready] action create (ceph-qa::ubuntu line 739) 2014-07-01 16:09:41,928.928 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:41-07:00] INFO: Chef Run complete in 28.698397815 seconds 2014-07-01 16:09:41,928.928 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:41-07:00] INFO: Running report handlers 2014-07-01 16:09:41,928.928 INFO:teuthology.orchestra.run.mira028.stdout:[2014-07-01T07:09:41-07:00] INFO: Report handlers complete 2014-07-01 16:09:42,025.025 INFO:teuthology.orchestra.run.mira028.stderr:+ cleanup 2014-07-01 16:09:42,025.025 INFO:teuthology.orchestra.run.mira028.stderr:+ rm -rf /tmp/solo-from-scratch.EiUJGCEOEa09 2014-07-01 16:09:42,048.048 INFO:teuthology.task.chef:Reconnecting after ceph-qa-chef run 2014-07-01 16:09:42,048.048 INFO:teuthology.misc:Re-opening connections... 2014-07-01 16:09:42,048.048 INFO:teuthology.misc:trying to connect to ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:09:42,049.049 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira079.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:09:45,572.572 INFO:teuthology.orchestra.run.mira079:Running: 'true' 2014-07-01 16:09:46,019.019 INFO:teuthology.misc:trying to connect to ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:09:46,021.021 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira028.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:09:49,473.473 INFO:teuthology.orchestra.run.mira028:Running: 'true' 2014-07-01 16:09:49,894.894 DEBUG:teuthology.misc:waited 7.84621787071 2014-07-01 16:09:50,894.894 INFO:teuthology.misc:trying to connect to ubuntu@mira043.front.sepia.ceph.com 2014-07-01 16:09:50,897.897 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'mira043.front.sepia.ceph.com', 'timeout': 60} 2014-07-01 16:09:54,856.856 INFO:teuthology.orchestra.run.mira043:Running: 'true' 2014-07-01 16:09:55,320.320 DEBUG:teuthology.misc:waited 13.2716059685 2014-07-01 16:09:56,318.318 INFO:teuthology.run_tasks:Running task clock.check... 2014-07-01 16:09:56,320.320 INFO:teuthology.task.clock:Checking initial clock skew... 2014-07-01 16:09:56,320.320 INFO:teuthology.orchestra.run.mira079:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 16:09:57,456.456 INFO:teuthology.orchestra.run.mira079.stdout: remote local st poll reach delay offset disp 2014-07-01 16:09:57,456.456 INFO:teuthology.orchestra.run.mira079.stdout:======================================================================= 2014-07-01 16:09:57,457.457 INFO:teuthology.orchestra.run.mira079.stdout:*clock3.dreamhos 10.214.135.128 1 16 7 0.00070 0.001406 0.01308 2014-07-01 16:09:57,458.458 INFO:teuthology.orchestra.run.mira043:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 16:09:58,234.234 INFO:teuthology.orchestra.run.mira043.stdout: remote local st poll reach delay offset disp 2014-07-01 16:09:58,235.235 INFO:teuthology.orchestra.run.mira043.stdout:======================================================================= 2014-07-01 16:09:58,235.235 INFO:teuthology.orchestra.run.mira043.stdout:*clock3.dreamhos 10.214.133.128 1 32 377 0.00038 -0.000001 0.03508 2014-07-01 16:09:58,235.235 INFO:teuthology.orchestra.run.mira028:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 16:09:59,155.155 INFO:teuthology.orchestra.run.mira028.stdout: remote local st poll reach delay offset disp 2014-07-01 16:09:59,156.156 INFO:teuthology.orchestra.run.mira028.stdout:======================================================================= 2014-07-01 16:09:59,156.156 INFO:teuthology.orchestra.run.mira028.stdout:*clock3.dreamhos 10.214.132.120 1 16 77 0.00064 -0.003899 0.01591 2014-07-01 16:09:59,158.158 INFO:teuthology.run_tasks:Running task install... 2014-07-01 16:09:59,159.159 DEBUG:teuthology.task.install:project ceph 2014-07-01 16:09:59,159.159 DEBUG:teuthology.task.install:config {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'branch': 'firefly'} 2014-07-01 16:09:59,159.159 INFO:teuthology.task.install:extra packages: [] 2014-07-01 16:09:59,160.160 INFO:teuthology.orchestra.run.mira079:Running: 'sudo lsb_release -is' 2014-07-01 16:09:59,673.673 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:09:59,673.673 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 16:09:59,675.675 INFO:teuthology.orchestra.run.mira079:Running: 'sudo apt-key list | grep Ceph' 2014-07-01 16:10:00,191.191 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:10:00,191.191 INFO:teuthology.orchestra.run.mira028:Running: 'sudo lsb_release -is' 2014-07-01 16:10:00,193.193 INFO:teuthology.orchestra.run.mira043:Running: 'sudo apt-key list | grep Ceph' 2014-07-01 16:10:00,455.455 INFO:teuthology.orchestra.run.mira079:Running: 'arch' 2014-07-01 16:10:00,905.905 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:10:00,906.906 INFO:teuthology.orchestra.run.mira028:Running: 'sudo apt-key list | grep Ceph' 2014-07-01 16:10:00,954.954 INFO:teuthology.orchestra.run.mira043:Running: 'arch' 2014-07-01 16:10:01,118.118 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -is' 2014-07-01 16:10:01,621.621 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -is' 2014-07-01 16:10:01,702.702 INFO:teuthology.orchestra.run.mira028:Running: 'arch' 2014-07-01 16:10:01,854.854 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -rs' 2014-07-01 16:10:02,363.363 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -is' 2014-07-01 16:10:02,367.367 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -rs' 2014-07-01 16:10:02,548.548 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -sc' 2014-07-01 16:10:03,028.028 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -sc' 2014-07-01 16:10:03,040.040 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -rs' 2014-07-01 16:10:03,204.204 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:03,205.205 INFO:teuthology.task.install:Installing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, librados2, librados2-dbg, librbd1, librbd1-dbg on remote deb x86_64 2014-07-01 16:10:03,205.205 INFO:teuthology.orchestra.run.mira079:Running: 'arch' 2014-07-01 16:10:03,640.640 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:03,641.641 INFO:teuthology.task.install:Installing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, librados2, librados2-dbg, librbd1, librbd1-dbg on remote deb x86_64 2014-07-01 16:10:03,641.641 INFO:teuthology.orchestra.run.mira043:Running: 'arch' 2014-07-01 16:10:03,649.649 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -sc' 2014-07-01 16:10:03,715.715 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -is' 2014-07-01 16:10:04,152.152 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -is' 2014-07-01 16:10:04,189.189 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:04,189.189 INFO:teuthology.task.install:Installing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, librados2, librados2-dbg, librbd1, librbd1-dbg on remote deb x86_64 2014-07-01 16:10:04,189.189 INFO:teuthology.orchestra.run.mira028:Running: 'arch' 2014-07-01 16:10:04,354.354 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -rs' 2014-07-01 16:10:04,730.730 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -is' 2014-07-01 16:10:04,765.765 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -rs' 2014-07-01 16:10:04,898.898 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -sc' 2014-07-01 16:10:05,304.304 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -rs' 2014-07-01 16:10:05,308.308 INFO:teuthology.orchestra.run.mira043:Running: 'lsb_release -sc' 2014-07-01 16:10:05,446.446 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:05,446.446 INFO:teuthology.orchestra.run.mira079:Running: 'sudo lsb_release -is' 2014-07-01 16:10:05,845.845 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -sc' 2014-07-01 16:10:05,848.848 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:05,849.849 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 16:10:06,002.002 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:10:06,003.003 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly 2014-07-01 16:10:06,003.003 INFO:teuthology.orchestra.run.mira079:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly/version' 2014-07-01 16:10:06,393.393 INFO:teuthology.task.install:config is {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac', 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': 'firefly', 'flavor': 'basic', 'extras': None} 2014-07-01 16:10:06,393.393 INFO:teuthology.orchestra.run.mira028:Running: 'sudo lsb_release -is' 2014-07-01 16:10:06,400.400 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:10:06,400.400 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly 2014-07-01 16:10:06,400.400 INFO:teuthology.orchestra.run.mira043:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly/version' 2014-07-01 16:10:06,521.521 INFO:teuthology.task.install:Package version is 0.80.1-50-gd43e711-1precise 2014-07-01 16:10:06,521.521 INFO:teuthology.orchestra.run.mira079:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly precise main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-07-01 16:10:06,917.917 INFO:teuthology.task.install:Package version is 0.80.1-50-gd43e711-1precise 2014-07-01 16:10:06,918.918 INFO:teuthology.orchestra.run.mira043:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly precise main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-07-01 16:10:06,947.947 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:10:06,947.947 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly 2014-07-01 16:10:06,947.947 INFO:teuthology.orchestra.run.mira028:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly/version' 2014-07-01 16:10:07,075.075 INFO:teuthology.orchestra.run.mira079:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.80.1-50-gd43e711-1precise ceph-dbg=0.80.1-50-gd43e711-1precise ceph-mds=0.80.1-50-gd43e711-1precise ceph-mds-dbg=0.80.1-50-gd43e711-1precise ceph-common=0.80.1-50-gd43e711-1precise ceph-common-dbg=0.80.1-50-gd43e711-1precise ceph-fuse=0.80.1-50-gd43e711-1precise ceph-fuse-dbg=0.80.1-50-gd43e711-1precise ceph-test=0.80.1-50-gd43e711-1precise ceph-test-dbg=0.80.1-50-gd43e711-1precise radosgw=0.80.1-50-gd43e711-1precise radosgw-dbg=0.80.1-50-gd43e711-1precise python-ceph=0.80.1-50-gd43e711-1precise libcephfs1=0.80.1-50-gd43e711-1precise libcephfs1-dbg=0.80.1-50-gd43e711-1precise librados2=0.80.1-50-gd43e711-1precise librados2-dbg=0.80.1-50-gd43e711-1precise librbd1=0.80.1-50-gd43e711-1precise librbd1-dbg=0.80.1-50-gd43e711-1precise' 2014-07-01 16:10:07,465.465 INFO:teuthology.task.install:Package version is 0.80.1-50-gd43e711-1precise 2014-07-01 16:10:07,465.465 INFO:teuthology.orchestra.run.mira028:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/ref/firefly precise main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-07-01 16:10:07,478.478 INFO:teuthology.orchestra.run.mira043:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.80.1-50-gd43e711-1precise ceph-dbg=0.80.1-50-gd43e711-1precise ceph-mds=0.80.1-50-gd43e711-1precise ceph-mds-dbg=0.80.1-50-gd43e711-1precise ceph-common=0.80.1-50-gd43e711-1precise ceph-common-dbg=0.80.1-50-gd43e711-1precise ceph-fuse=0.80.1-50-gd43e711-1precise ceph-fuse-dbg=0.80.1-50-gd43e711-1precise ceph-test=0.80.1-50-gd43e711-1precise ceph-test-dbg=0.80.1-50-gd43e711-1precise radosgw=0.80.1-50-gd43e711-1precise radosgw-dbg=0.80.1-50-gd43e711-1precise python-ceph=0.80.1-50-gd43e711-1precise libcephfs1=0.80.1-50-gd43e711-1precise libcephfs1-dbg=0.80.1-50-gd43e711-1precise librados2=0.80.1-50-gd43e711-1precise librados2-dbg=0.80.1-50-gd43e711-1precise librbd1=0.80.1-50-gd43e711-1precise librbd1-dbg=0.80.1-50-gd43e711-1precise' 2014-07-01 16:10:08,043.043 INFO:teuthology.orchestra.run.mira028:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.80.1-50-gd43e711-1precise ceph-dbg=0.80.1-50-gd43e711-1precise ceph-mds=0.80.1-50-gd43e711-1precise ceph-mds-dbg=0.80.1-50-gd43e711-1precise ceph-common=0.80.1-50-gd43e711-1precise ceph-common-dbg=0.80.1-50-gd43e711-1precise ceph-fuse=0.80.1-50-gd43e711-1precise ceph-fuse-dbg=0.80.1-50-gd43e711-1precise ceph-test=0.80.1-50-gd43e711-1precise ceph-test-dbg=0.80.1-50-gd43e711-1precise radosgw=0.80.1-50-gd43e711-1precise radosgw-dbg=0.80.1-50-gd43e711-1precise python-ceph=0.80.1-50-gd43e711-1precise libcephfs1=0.80.1-50-gd43e711-1precise libcephfs1-dbg=0.80.1-50-gd43e711-1precise librados2=0.80.1-50-gd43e711-1precise librados2-dbg=0.80.1-50-gd43e711-1precise librbd1=0.80.1-50-gd43e711-1precise librbd1-dbg=0.80.1-50-gd43e711-1precise' 2014-07-01 16:10:11,610.610 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: downgrading librbd1-dbg from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:11,952.952 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: downgrading librados2-dbg from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. Extracting templates from packages: 100% 2014-07-01 16:10:12,710.710 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: downgrading librbd1-dbg from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:12,957.957 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: downgrading librbd1 from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:13,072.072 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: downgrading librados2-dbg from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:13,817.817 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: downgrading librbd1 from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:14,104.104 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: downgrading librados2 from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:14,619.619 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: downgrading librados2 from 0.81-365-g0aed380-1precise to 0.80.1-50-gd43e711-1precise. 2014-07-01 16:10:59,382.382 DEBUG:teuthology.parallel:result is None 2014-07-01 16:11:10,073.073 DEBUG:teuthology.parallel:result is None 2014-07-01 16:11:23,785.785 DEBUG:teuthology.parallel:result is None 2014-07-01 16:11:23,787.787 INFO:teuthology.run_tasks:Running task ceph... 2014-07-01 16:11:23,788.788 INFO:teuthology.task.ceph:Making ceph log dir writeable by non-root... 2014-07-01 16:11:23,788.788 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod 777 /var/log/ceph' 2014-07-01 16:11:24,158.158 INFO:teuthology.orchestra.run.mira043:Running: 'sudo chmod 777 /var/log/ceph' 2014-07-01 16:11:24,495.495 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 777 /var/log/ceph' 2014-07-01 16:11:24,845.845 INFO:teuthology.task.ceph:Disabling ceph logrotate... 2014-07-01 16:11:24,845.845 INFO:teuthology.orchestra.run.mira028:Running: 'sudo rm -f -- /etc/logrotate.d/ceph' 2014-07-01 16:11:25,213.213 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -f -- /etc/logrotate.d/ceph' 2014-07-01 16:11:25,546.546 INFO:teuthology.orchestra.run.mira079:Running: 'sudo rm -f -- /etc/logrotate.d/ceph' 2014-07-01 16:11:25,930.930 INFO:teuthology.task.ceph:Creating extra log directories... 2014-07-01 16:11:25,931.931 INFO:teuthology.orchestra.run.mira028:Running: 'sudo install -d -m0755 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger' 2014-07-01 16:11:26,300.300 INFO:teuthology.orchestra.run.mira043:Running: 'sudo install -d -m0755 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger' 2014-07-01 16:11:26,669.669 INFO:teuthology.orchestra.run.mira079:Running: 'sudo install -d -m0755 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger' 2014-07-01 16:11:27,056.056 INFO:teuthology.task.ceph:Shipping valgrind.supp... 2014-07-01 16:11:27,056.056 INFO:teuthology.orchestra.run.mira079:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/valgrind.supp' 2014-07-01 16:11:27,991.991 INFO:teuthology.orchestra.run.mira043:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/valgrind.supp' 2014-07-01 16:11:28,728.728 INFO:teuthology.orchestra.run.mira028:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/valgrind.supp' 2014-07-01 16:11:29,466.466 INFO:teuthology.task.ceph:Shipping 'daemon-helper'... 2014-07-01 16:11:29,466.466 INFO:teuthology.orchestra.run.mira079:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/daemon-helper' 2014-07-01 16:11:30,157.157 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod a=rx -- /usr/bin/daemon-helper' 2014-07-01 16:11:30,712.712 INFO:teuthology.orchestra.run.mira043:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/daemon-helper' 2014-07-01 16:11:31,408.408 INFO:teuthology.orchestra.run.mira043:Running: 'sudo chmod a=rx -- /usr/bin/daemon-helper' 2014-07-01 16:11:31,966.966 INFO:teuthology.orchestra.run.mira028:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/daemon-helper' 2014-07-01 16:11:32,659.659 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod a=rx -- /usr/bin/daemon-helper' 2014-07-01 16:11:33,213.213 INFO:teuthology.task.ceph:Shipping 'adjust-ulimits'... 2014-07-01 16:11:33,214.214 INFO:teuthology.orchestra.run.mira079:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/adjust-ulimits' 2014-07-01 16:11:33,928.928 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod a=rx -- /usr/bin/adjust-ulimits' 2014-07-01 16:11:34,483.483 INFO:teuthology.orchestra.run.mira043:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/adjust-ulimits' 2014-07-01 16:11:35,202.202 INFO:teuthology.orchestra.run.mira043:Running: 'sudo chmod a=rx -- /usr/bin/adjust-ulimits' 2014-07-01 16:11:35,758.758 INFO:teuthology.orchestra.run.mira028:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/adjust-ulimits' 2014-07-01 16:11:36,508.508 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod a=rx -- /usr/bin/adjust-ulimits' 2014-07-01 16:11:37,063.063 INFO:teuthology.task.ceph:Shipping 'kcon_most'... 2014-07-01 16:11:37,064.064 INFO:teuthology.orchestra.run.mira079:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/kcon_most' 2014-07-01 16:11:37,787.787 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod a=rx -- /usr/bin/kcon_most' 2014-07-01 16:11:38,386.386 INFO:teuthology.orchestra.run.mira043:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/kcon_most' 2014-07-01 16:11:39,160.160 INFO:teuthology.orchestra.run.mira043:Running: 'sudo chmod a=rx -- /usr/bin/kcon_most' 2014-07-01 16:11:39,717.717 INFO:teuthology.orchestra.run.mira028:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/kcon_most' 2014-07-01 16:11:40,468.468 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod a=rx -- /usr/bin/kcon_most' 2014-07-01 16:11:41,022.022 INFO:teuthology.task.ceph:Creating ceph cluster... 2014-07-01 16:11:41,023.023 INFO:teuthology.orchestra.run.mira028:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/data' 2014-07-01 16:11:41,564.564 INFO:teuthology.orchestra.run.mira043:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/data' 2014-07-01 16:11:41,900.900 INFO:teuthology.orchestra.run.mira079:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/data' 2014-07-01 16:11:42,245.245 INFO:teuthology.orchestra.run.mira028:Running: 'sudo install -d -m0777 -- /var/run/ceph' 2014-07-01 16:11:42,581.581 INFO:teuthology.orchestra.run.mira043:Running: 'sudo install -d -m0777 -- /var/run/ceph' 2014-07-01 16:11:42,918.918 INFO:teuthology.orchestra.run.mira079:Running: 'sudo install -d -m0777 -- /var/run/ceph' 2014-07-01 16:11:44,157.157 INFO:teuthology.orchestra.run.mira079:Running: 'ls /dev/[sv]d?' 2014-07-01 16:11:44,500.500 DEBUG:teuthology.misc:devs=['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', '/dev/sdg'] 2014-07-01 16:11:44,500.500 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sda && sudo dd if=/dev/sda of=/dev/null count=1 && ! mount | grep -q /dev/sda' 2014-07-01 16:11:45,011.011 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sda' 2014-07-01 16:11:45,012.012 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:45,012.012 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3136 Links: 1 Device type: 8,0 2014-07-01 16:11:45,012.012 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:45,013.013 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 06:51:13.030900933 -0700 2014-07-01 16:11:45,013.013 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.638900941 -0700 2014-07-01 16:11:45,013.013 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.638900941 -0700 2014-07-01 16:11:45,013.013 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:45,022.022 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:45,022.022 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:45,022.022 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 7.9498e-05 s, 6.4 MB/s 2014-07-01 16:11:45,027.027 DEBUG:teuthology.misc:get_scratch_devices: /dev/sda is in use 2014-07-01 16:11:45,028.028 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sdb && sudo dd if=/dev/sdb of=/dev/null count=1 && ! mount | grep -q /dev/sdb' 2014-07-01 16:11:45,571.571 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sdb' 2014-07-01 16:11:45,571.571 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:45,572.572 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3142 Links: 1 Device type: 8,10 2014-07-01 16:11:45,572.572 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:45,572.572 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 06:51:13.038900933 -0700 2014-07-01 16:11:45,572.572 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:45,573.573 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:45,573.573 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:45,579.579 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:45,579.579 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:45,579.579 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.000530316 s, 965 kB/s 2014-07-01 16:11:45,583.583 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sdc && sudo dd if=/dev/sdc of=/dev/null count=1 && ! mount | grep -q /dev/sdc' 2014-07-01 16:11:46,126.126 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sdc' 2014-07-01 16:11:46,126.126 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:46,126.126 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3144 Links: 1 Device type: 8,20 2014-07-01 16:11:46,127.127 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:46,127.127 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 06:51:13.038900933 -0700 2014-07-01 16:11:46,127.127 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.650900941 -0700 2014-07-01 16:11:46,127.127 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.650900941 -0700 2014-07-01 16:11:46,128.128 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:46,136.136 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:46,137.137 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:46,137.137 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.00034785 s, 1.5 MB/s 2014-07-01 16:11:46,143.143 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sdd && sudo dd if=/dev/sdd of=/dev/null count=1 && ! mount | grep -q /dev/sdd' 2014-07-01 16:11:46,688.688 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sdd' 2014-07-01 16:11:46,688.688 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:46,688.688 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3145 Links: 1 Device type: 8,30 2014-07-01 16:11:46,688.688 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:46,689.689 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 07:09:10.196012308 -0700 2014-07-01 16:11:46,689.689 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:46,689.689 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:46,689.689 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:46,699.699 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:46,699.699 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:46,699.699 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.000607914 s, 842 kB/s 2014-07-01 16:11:46,705.705 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sde && sudo dd if=/dev/sde of=/dev/null count=1 && ! mount | grep -q /dev/sde' 2014-07-01 16:11:47,250.250 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sde' 2014-07-01 16:11:47,250.250 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:47,251.251 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3146 Links: 1 Device type: 8,40 2014-07-01 16:11:47,251.251 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:47,251.251 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 07:09:10.288012775 -0700 2014-07-01 16:11:47,251.251 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:47,252.252 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:47,252.252 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:47,287.287 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:47,288.288 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:47,288.288 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.0266763 s, 19.2 kB/s 2014-07-01 16:11:47,294.294 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sdf && sudo dd if=/dev/sdf of=/dev/null count=1 && ! mount | grep -q /dev/sdf' 2014-07-01 16:11:47,791.791 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sdf' 2014-07-01 16:11:47,791.791 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:47,791.791 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3147 Links: 1 Device type: 8,50 2014-07-01 16:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 07:09:10.400013349 -0700 2014-07-01 16:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.650900941 -0700 2014-07-01 16:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.650900941 -0700 2014-07-01 16:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:47,801.801 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:47,801.801 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:47,801.801 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.000616657 s, 830 kB/s 2014-07-01 16:11:47,807.807 INFO:teuthology.orchestra.run.mira079:Running: 'stat /dev/sdg && sudo dd if=/dev/sdg of=/dev/null count=1 && ! mount | grep -q /dev/sdg' 2014-07-01 16:11:48,351.351 INFO:teuthology.orchestra.run.mira079.stdout: File: `/dev/sdg' 2014-07-01 16:11:48,351.351 INFO:teuthology.orchestra.run.mira079.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:48,352.352 INFO:teuthology.orchestra.run.mira079.stdout:Device: 5h/5d Inode: 3143 Links: 1 Device type: 8,60 2014-07-01 16:11:48,352.352 INFO:teuthology.orchestra.run.mira079.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:48,352.352 INFO:teuthology.orchestra.run.mira079.stdout:Access: 2014-07-01 07:09:09.852010516 -0700 2014-07-01 16:11:48,352.352 INFO:teuthology.orchestra.run.mira079.stdout:Modify: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:48,352.352 INFO:teuthology.orchestra.run.mira079.stdout:Change: 2014-07-01 06:51:12.646900941 -0700 2014-07-01 16:11:48,353.353 INFO:teuthology.orchestra.run.mira079.stdout: Birth: - 2014-07-01 16:11:48,381.381 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records in 2014-07-01 16:11:48,381.381 INFO:teuthology.orchestra.run.mira079.stderr:1+0 records out 2014-07-01 16:11:48,381.381 INFO:teuthology.orchestra.run.mira079.stderr:512 bytes (512 B) copied, 0.0221512 s, 23.1 kB/s 2014-07-01 16:11:48,387.387 INFO:teuthology.task.ceph:fs option selected, checking for scratch devs 2014-07-01 16:11:48,387.387 INFO:teuthology.task.ceph:found devs: ['/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', '/dev/sdg'] 2014-07-01 16:11:48,387.387 INFO:teuthology.orchestra.run.mira079:Running: "ls -l '/dev/disk/by-id/wwn-*'" 2014-07-01 16:11:48,901.901 INFO:teuthology.orchestra.run.mira079.stderr:ls: cannot access /dev/disk/by-id/wwn-*: No such file or directory 2014-07-01 16:11:48,902.902 ERROR:teuthology.misc:Failed to get wwn devices! Using /dev/sd* devices... 2014-07-01 16:11:48,903.903 INFO:teuthology.task.ceph:dev map: {'1': '/dev/sdg', '0': '/dev/sdf', '2': '/dev/sdd'} 2014-07-01 16:11:49,583.583 INFO:teuthology.orchestra.run.mira028:Running: 'ls /dev/[sv]d?' 2014-07-01 16:11:49,928.928 DEBUG:teuthology.misc:devs=['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh'] 2014-07-01 16:11:49,928.928 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sda && sudo dd if=/dev/sda of=/dev/null count=1 && ! mount | grep -q /dev/sda' 2014-07-01 16:11:50,440.440 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sda' 2014-07-01 16:11:50,440.440 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:50,440.440 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 3136 Links: 1 Device type: 8,0 2014-07-01 16:11:50,441.441 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:50,441.441 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 06:51:07.327385920 -0700 2014-07-01 16:11:50,441.441 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:07.311385920 -0700 2014-07-01 16:11:50,441.441 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:07.311385920 -0700 2014-07-01 16:11:50,441.441 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:50,450.450 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:50,450.450 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:50,451.451 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 9.6032e-05 s, 5.3 MB/s 2014-07-01 16:11:50,455.455 DEBUG:teuthology.misc:get_scratch_devices: /dev/sda is in use 2014-07-01 16:11:50,455.455 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdb && sudo dd if=/dev/sdb of=/dev/null count=1 && ! mount | grep -q /dev/sdb' 2014-07-01 16:11:51,000.000 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdb' 2014-07-01 16:11:51,000.000 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:51,001.001 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 10468 Links: 1 Device type: 8,10 2014-07-01 16:11:51,001.001 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:51,001.001 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 06:51:06.803385930 -0700 2014-07-01 16:11:51,001.001 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:51,002.002 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:51,002.002 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:51,011.011 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:51,011.011 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:51,012.012 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.000390082 s, 1.3 MB/s 2014-07-01 16:11:51,017.017 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdc && sudo dd if=/dev/sdc of=/dev/null count=1 && ! mount | grep -q /dev/sdc' 2014-07-01 16:11:51,562.562 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdc' 2014-07-01 16:11:51,563.563 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:51,563.563 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 12358 Links: 1 Device type: 8,20 2014-07-01 16:11:51,563.563 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:51,563.563 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 06:51:06.827385930 -0700 2014-07-01 16:11:51,564.564 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.555385935 -0700 2014-07-01 16:11:51,564.564 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.555385935 -0700 2014-07-01 16:11:51,564.564 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:51,573.573 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:51,573.573 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:51,574.574 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.000366999 s, 1.4 MB/s 2014-07-01 16:11:51,579.579 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdd && sudo dd if=/dev/sdd of=/dev/null count=1 && ! mount | grep -q /dev/sdd' 2014-07-01 16:11:52,122.122 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdd' 2014-07-01 16:11:52,122.122 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:52,122.122 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 12360 Links: 1 Device type: 8,30 2014-07-01 16:11:52,123.123 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:52,123.123 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 06:51:06.803385930 -0700 2014-07-01 16:11:52,123.123 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.555385935 -0700 2014-07-01 16:11:52,123.123 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.555385935 -0700 2014-07-01 16:11:52,123.123 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:52,132.132 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:52,132.132 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:52,132.132 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.000355771 s, 1.4 MB/s 2014-07-01 16:11:52,138.138 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sde && sudo dd if=/dev/sde of=/dev/null count=1 && ! mount | grep -q /dev/sde' 2014-07-01 16:11:52,680.680 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sde' 2014-07-01 16:11:52,680.680 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:52,681.681 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 3143 Links: 1 Device type: 8,40 2014-07-01 16:11:52,681.681 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:52,681.681 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 07:09:10.873001143 -0700 2014-07-01 16:11:52,681.681 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.551385935 -0700 2014-07-01 16:11:52,681.681 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.551385935 -0700 2014-07-01 16:11:52,682.682 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:52,704.704 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:52,704.704 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:52,704.704 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.0127292 s, 40.2 kB/s 2014-07-01 16:11:52,710.710 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdf && sudo dd if=/dev/sdf of=/dev/null count=1 && ! mount | grep -q /dev/sdf' 2014-07-01 16:11:53,223.223 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdf' 2014-07-01 16:11:53,223.223 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:53,224.224 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 1228 Links: 1 Device type: 8,50 2014-07-01 16:11:53,224.224 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:53,224.224 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 07:09:10.973004342 -0700 2014-07-01 16:11:53,224.224 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:53,225.225 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:53,225.225 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:53,236.236 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:53,236.236 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:53,236.236 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.00342362 s, 150 kB/s 2014-07-01 16:11:53,242.242 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdg && sudo dd if=/dev/sdg of=/dev/null count=1 && ! mount | grep -q /dev/sdg' 2014-07-01 16:11:53,788.788 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdg' 2014-07-01 16:11:53,788.788 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:53,789.789 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 10474 Links: 1 Device type: 8,60 2014-07-01 16:11:53,789.789 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:53,789.789 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 07:09:11.093008186 -0700 2014-07-01 16:11:53,790.790 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.655385933 -0700 2014-07-01 16:11:53,790.790 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.655385933 -0700 2014-07-01 16:11:53,790.790 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:53,803.803 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:53,803.803 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:53,803.803 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.00406716 s, 126 kB/s 2014-07-01 16:11:53,807.807 INFO:teuthology.orchestra.run.mira028:Running: 'stat /dev/sdh && sudo dd if=/dev/sdh of=/dev/null count=1 && ! mount | grep -q /dev/sdh' 2014-07-01 16:11:54,354.354 INFO:teuthology.orchestra.run.mira028.stdout: File: `/dev/sdh' 2014-07-01 16:11:54,355.355 INFO:teuthology.orchestra.run.mira028.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-07-01 16:11:54,355.355 INFO:teuthology.orchestra.run.mira028.stdout:Device: 5h/5d Inode: 12362 Links: 1 Device type: 8,70 2014-07-01 16:11:54,355.355 INFO:teuthology.orchestra.run.mira028.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-07-01 16:11:54,356.356 INFO:teuthology.orchestra.run.mira028.stdout:Access: 2014-07-01 07:09:11.245013053 -0700 2014-07-01 16:11:54,356.356 INFO:teuthology.orchestra.run.mira028.stdout:Modify: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:54,356.356 INFO:teuthology.orchestra.run.mira028.stdout:Change: 2014-07-01 06:51:06.527385936 -0700 2014-07-01 16:11:54,356.356 INFO:teuthology.orchestra.run.mira028.stdout: Birth: - 2014-07-01 16:11:54,370.370 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records in 2014-07-01 16:11:54,370.370 INFO:teuthology.orchestra.run.mira028.stderr:1+0 records out 2014-07-01 16:11:54,370.370 INFO:teuthology.orchestra.run.mira028.stderr:512 bytes (512 B) copied, 0.00769751 s, 66.5 kB/s 2014-07-01 16:11:54,374.374 INFO:teuthology.task.ceph:fs option selected, checking for scratch devs 2014-07-01 16:11:54,374.374 INFO:teuthology.task.ceph:found devs: ['/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh'] 2014-07-01 16:11:54,375.375 INFO:teuthology.orchestra.run.mira028:Running: "ls -l '/dev/disk/by-id/wwn-*'" 2014-07-01 16:11:54,919.919 INFO:teuthology.orchestra.run.mira028.stderr:ls: cannot access /dev/disk/by-id/wwn-*: No such file or directory 2014-07-01 16:11:54,920.920 ERROR:teuthology.misc:Failed to get wwn devices! Using /dev/sd* devices... 2014-07-01 16:11:54,920.920 INFO:teuthology.task.ceph:dev map: {'3': '/dev/sdh', '5': '/dev/sdg', '4': '/dev/sdf'} 2014-07-01 16:11:54,921.921 INFO:teuthology.task.ceph:Generating config... 2014-07-01 16:11:54,925.925 INFO:teuthology.task.ceph:[mon] debug mon = 20 2014-07-01 16:11:54,925.925 INFO:teuthology.task.ceph:[mon] debug paxos = 20 2014-07-01 16:11:54,925.925 INFO:teuthology.task.ceph:[mon] debug ms = 1 2014-07-01 16:11:54,926.926 INFO:teuthology.task.ceph:[osd] debug osd = 20 2014-07-01 16:11:54,926.926 INFO:teuthology.task.ceph:[osd] debug ms = 1 2014-07-01 16:11:54,926.926 INFO:teuthology.task.ceph:[osd] debug journal = 20 2014-07-01 16:11:54,926.926 INFO:teuthology.task.ceph:[osd] debug filestore = 20 2014-07-01 16:11:54,927.927 INFO:teuthology.task.ceph:Setting up mon.a... 2014-07-01 16:11:54,927.927 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring' 2014-07-01 16:11:55,317.317 INFO:teuthology.orchestra.run.mira079.stdout:creating /etc/ceph/ceph.keyring 2014-07-01 16:11:55,321.321 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring' 2014-07-01 16:11:55,837.837 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-07-01 16:11:56,355.355 DEBUG:teuthology.misc:Ceph mon addresses: [('a', '10.214.135.128:6789'), ('c', '10.214.132.120:6789'), ('b', '10.214.135.128:6790')] 2014-07-01 16:11:56,356.356 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool --create --clobber --add a 10.214.135.128:6789 --add c 10.214.132.120:6789 --add b 10.214.135.128:6790 --print /home/ubuntu/cephtest/monmap' 2014-07-01 16:11:56,922.922 INFO:teuthology.task.ceph:Writing ceph.conf for FSID d0c0ad56-ac30-428a-b414-dd3ce1a5627d... 2014-07-01 16:11:56,924.924 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf' 2014-07-01 16:11:57,262.262 INFO:teuthology.orchestra.run.mira043:Running: 'sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf' 2014-07-01 16:11:57,601.601 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf' 2014-07-01 16:11:57,938.938 WARNING:teuthology.task.ceph:writes: 2014-07-01 16:11:58,322.322 INFO:teuthology.task.ceph:Creating admin key on mon.a... 2014-07-01 16:11:58,322.322 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds allow /etc/ceph/ceph.keyring" 2014-07-01 16:11:58,933.933 INFO:teuthology.task.ceph:Copying monmap to all nodes... 2014-07-01 16:12:02,618.618 INFO:teuthology.task.ceph:Sending monmap to node ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:12:02,618.618 INFO:teuthology.orchestra.run.mira079:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.keyring && sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-07-01 16:12:03,305.305 INFO:teuthology.orchestra.run.mira079:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:04,223.223 INFO:teuthology.task.ceph:Sending monmap to node ubuntu@mira043.front.sepia.ceph.com 2014-07-01 16:12:04,224.224 INFO:teuthology.orchestra.run.mira043:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.keyring && sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-07-01 16:12:04,986.986 INFO:teuthology.orchestra.run.mira043:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:05,904.904 INFO:teuthology.task.ceph:Sending monmap to node ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:12:05,905.905 INFO:teuthology.orchestra.run.mira028:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.keyring && sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-07-01 16:12:06,619.619 INFO:teuthology.orchestra.run.mira028:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:07,534.534 INFO:teuthology.task.ceph:Setting up mon nodes... 2014-07-01 16:12:07,535.535 INFO:teuthology.orchestra.run.mira028:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage osdmaptool -c /etc/ceph/ceph.conf --clobber --createsimple 6 /home/ubuntu/cephtest/osdmap --pg_bits 2 --pgp_bits 4' 2014-07-01 16:12:08,075.075 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage osdmaptool -c /etc/ceph/ceph.conf --clobber --createsimple 6 /home/ubuntu/cephtest/osdmap --pg_bits 2 --pgp_bits 4' 2014-07-01 16:12:08,109.109 INFO:teuthology.orchestra.run.mira028.stderr:osdmaptool: osdmap file '/home/ubuntu/cephtest/osdmap' 2014-07-01 16:12:08,110.110 INFO:teuthology.orchestra.run.mira028.stdout:osdmaptool: writing epoch 1 to /home/ubuntu/cephtest/osdmap 2014-07-01 16:12:08,474.474 INFO:teuthology.orchestra.run.mira079.stderr:osdmaptool: osdmap file '/home/ubuntu/cephtest/osdmap' 2014-07-01 16:12:08,475.475 INFO:teuthology.orchestra.run.mira079.stdout:osdmaptool: writing epoch 1 to /home/ubuntu/cephtest/osdmap 2014-07-01 16:12:08,477.477 INFO:teuthology.task.ceph:Setting up mds nodes... 2014-07-01 16:12:08,477.477 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/mds/ceph-a && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.a /var/lib/ceph/mds/ceph-a/keyring' 2014-07-01 16:12:09,055.055 INFO:teuthology.orchestra.run.mira079.stdout:creating /var/lib/ceph/mds/ceph-a/keyring 2014-07-01 16:12:09,060.060 INFO:teuthology.task.ceph_client:Setting up client nodes... 2014-07-01 16:12:09,061.061 INFO:teuthology.orchestra.run.mira043:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring' 2014-07-01 16:12:09,478.478 INFO:teuthology.orchestra.run.mira043.stdout:creating /etc/ceph/ceph.client.0.keyring 2014-07-01 16:12:09,488.488 INFO:teuthology.task.ceph:Running mkfs on osd nodes... 2014-07-01 16:12:09,488.488 INFO:teuthology.task.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@mira079.front.sepia.ceph.com'): {'1': '/dev/sdg', '0': '/dev/sdf', '2': '/dev/sdd'}, Remote(name='ubuntu@mira028.front.sepia.ceph.com'): {'3': '/dev/sdh', '5': '/dev/sdg', '4': '/dev/sdf'}} 2014-07-01 16:12:09,488.488 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-0' 2014-07-01 16:12:09,839.839 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:09,839.839 INFO:teuthology.task.ceph:0 2014-07-01 16:12:09,840.840 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdf on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:12:09,840.840 INFO:teuthology.orchestra.run.mira079:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdf' 2014-07-01 16:12:15,975.975 INFO:teuthology.orchestra.run.mira079.stdout:meta-data=/dev/sdf isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:12:15,976.976 INFO:teuthology.orchestra.run.mira079.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:12:15,977.977 INFO:teuthology.task.ceph:mount /dev/sdf on ubuntu@mira079.front.sepia.ceph.com -o noatime 2014-07-01 16:12:15,977.977 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mount -t xfs -o noatime /dev/sdf /var/lib/ceph/osd/ceph-0' 2014-07-01 16:12:16,625.625 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-1' 2014-07-01 16:12:17,041.041 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:17,041.041 INFO:teuthology.task.ceph:1 2014-07-01 16:12:17,041.041 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdg on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:12:17,041.041 INFO:teuthology.orchestra.run.mira079:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdg' 2014-07-01 16:12:26,574.574 INFO:teuthology.orchestra.run.mira079.stdout:meta-data=/dev/sdg isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:12:26,574.574 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:12:26,575.575 INFO:teuthology.orchestra.run.mira079.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:12:26,575.575 INFO:teuthology.orchestra.run.mira079.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:12:26,575.575 INFO:teuthology.orchestra.run.mira079.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:12:26,575.575 INFO:teuthology.orchestra.run.mira079.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:12:26,576.576 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:12:26,576.576 INFO:teuthology.orchestra.run.mira079.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:12:26,576.576 INFO:teuthology.task.ceph:mount /dev/sdg on ubuntu@mira079.front.sepia.ceph.com -o noatime 2014-07-01 16:12:26,577.577 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mount -t xfs -o noatime /dev/sdg /var/lib/ceph/osd/ceph-1' 2014-07-01 16:12:27,406.406 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-2' 2014-07-01 16:12:27,930.930 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:27,931.931 INFO:teuthology.task.ceph:2 2014-07-01 16:12:27,931.931 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdd on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:12:27,932.932 INFO:teuthology.orchestra.run.mira079:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdd' 2014-07-01 16:12:34,068.068 INFO:teuthology.orchestra.run.mira079.stdout:meta-data=/dev/sdd isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:12:34,068.068 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:12:34,068.068 INFO:teuthology.orchestra.run.mira079.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:12:34,069.069 INFO:teuthology.orchestra.run.mira079.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:12:34,069.069 INFO:teuthology.orchestra.run.mira079.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:12:34,069.069 INFO:teuthology.orchestra.run.mira079.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:12:34,069.069 INFO:teuthology.orchestra.run.mira079.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:12:34,070.070 INFO:teuthology.orchestra.run.mira079.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:12:34,070.070 INFO:teuthology.task.ceph:mount /dev/sdd on ubuntu@mira079.front.sepia.ceph.com -o noatime 2014-07-01 16:12:34,070.070 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mount -t xfs -o noatime /dev/sdd /var/lib/ceph/osd/ceph-2' 2014-07-01 16:12:34,696.696 INFO:teuthology.orchestra.run.mira079:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:35,189.189 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.099206 7fbb41002780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:35,199.199 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.110408 7fbb41002780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:35,200.200 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.111351 7fbb41002780 -1 filestore(/var/lib/ceph/osd/ceph-0) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:12:35,215.215 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.124739 7fbb41002780 -1 created object store /var/lib/ceph/osd/ceph-0 journal /var/lib/ceph/osd/ceph-0/journal for osd.0 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:12:35,215.215 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.124799 7fbb41002780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2014-07-01 16:12:35,215.215 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.124971 7fbb41002780 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2014-07-01 16:12:35,217.217 INFO:teuthology.orchestra.run.mira079:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:35,768.768 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.678182 7fcd991f2780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:35,776.776 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.687771 7fcd991f2780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:35,777.777 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.688714 7fcd991f2780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:12:35,823.823 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.733622 7fcd991f2780 -1 created object store /var/lib/ceph/osd/ceph-1 journal /var/lib/ceph/osd/ceph-1/journal for osd.1 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:12:35,823.823 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.733692 7fcd991f2780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2014-07-01 16:12:35,823.823 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:35.733864 7fcd991f2780 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2014-07-01 16:12:35,826.826 INFO:teuthology.orchestra.run.mira079:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:12:36,309.309 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.219295 7f74014f6780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:36,315.315 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.226143 7f74014f6780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:12:36,316.316 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.227074 7f74014f6780 -1 filestore(/var/lib/ceph/osd/ceph-2) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:12:36,375.375 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.285980 7f74014f6780 -1 created object store /var/lib/ceph/osd/ceph-2 journal /var/lib/ceph/osd/ceph-2/journal for osd.2 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:12:36,376.376 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.286053 7f74014f6780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2014-07-01 16:12:36,376.376 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:12:36.286228 7f74014f6780 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2014-07-01 16:12:36,379.379 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-3' 2014-07-01 16:12:36,767.767 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:36,767.767 INFO:teuthology.task.ceph:3 2014-07-01 16:12:36,767.767 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdh on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:12:36,768.768 INFO:teuthology.orchestra.run.mira028:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdh' 2014-07-01 16:12:44,886.886 INFO:teuthology.orchestra.run.mira028.stdout:meta-data=/dev/sdh isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:12:44,886.886 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:12:44,886.886 INFO:teuthology.orchestra.run.mira028.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:12:44,887.887 INFO:teuthology.orchestra.run.mira028.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:12:44,887.887 INFO:teuthology.orchestra.run.mira028.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:12:44,887.887 INFO:teuthology.orchestra.run.mira028.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:12:44,887.887 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:12:44,888.888 INFO:teuthology.orchestra.run.mira028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:12:44,888.888 INFO:teuthology.task.ceph:mount /dev/sdh on ubuntu@mira028.front.sepia.ceph.com -o noatime 2014-07-01 16:12:44,888.888 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mount -t xfs -o noatime /dev/sdh /var/lib/ceph/osd/ceph-3' 2014-07-01 16:12:46,894.894 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-4' 2014-07-01 16:12:47,417.417 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:47,417.417 INFO:teuthology.task.ceph:4 2014-07-01 16:12:47,418.418 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdf on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:12:47,418.418 INFO:teuthology.orchestra.run.mira028:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdf' 2014-07-01 16:12:53,939.939 INFO:teuthology.orchestra.run.mira028.stdout:meta-data=/dev/sdf isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:12:53,940.940 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:12:53,940.940 INFO:teuthology.orchestra.run.mira028.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:12:53,940.940 INFO:teuthology.orchestra.run.mira028.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:12:53,940.940 INFO:teuthology.orchestra.run.mira028.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:12:53,940.940 INFO:teuthology.orchestra.run.mira028.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:12:53,941.941 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:12:53,941.941 INFO:teuthology.orchestra.run.mira028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:12:53,943.943 INFO:teuthology.task.ceph:mount /dev/sdf on ubuntu@mira028.front.sepia.ceph.com -o noatime 2014-07-01 16:12:53,944.944 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mount -t xfs -o noatime /dev/sdf /var/lib/ceph/osd/ceph-4' 2014-07-01 16:12:54,568.568 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-5' 2014-07-01 16:12:55,018.018 INFO:teuthology.task.ceph:{} 2014-07-01 16:12:55,018.018 INFO:teuthology.task.ceph:5 2014-07-01 16:12:55,019.019 INFO:teuthology.task.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/sdg on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:12:55,019.019 INFO:teuthology.orchestra.run.mira028:Running: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/sdg' 2014-07-01 16:13:12,295.295 INFO:teuthology.orchestra.run.mira028.stdout:meta-data=/dev/sdg isize=2048 agcount=4, agsize=61047662 blks 2014-07-01 16:13:12,295.295 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 attr=2, projid32bit=0 2014-07-01 16:13:12,296.296 INFO:teuthology.orchestra.run.mira028.stdout:data = bsize=4096 blocks=244190646, imaxpct=25 2014-07-01 16:13:12,296.296 INFO:teuthology.orchestra.run.mira028.stdout: = sunit=0 swidth=0 blks 2014-07-01 16:13:12,296.296 INFO:teuthology.orchestra.run.mira028.stdout:naming =version 2 bsize=4096 ascii-ci=0 2014-07-01 16:13:12,296.296 INFO:teuthology.orchestra.run.mira028.stdout:log =internal log bsize=4096 blocks=119233, version=2 2014-07-01 16:13:12,297.297 INFO:teuthology.orchestra.run.mira028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2014-07-01 16:13:12,297.297 INFO:teuthology.orchestra.run.mira028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2014-07-01 16:13:12,297.297 INFO:teuthology.task.ceph:mount /dev/sdg on ubuntu@mira028.front.sepia.ceph.com -o noatime 2014-07-01 16:13:12,298.298 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mount -t xfs -o noatime /dev/sdg /var/lib/ceph/osd/ceph-5' 2014-07-01 16:13:12,933.933 INFO:teuthology.orchestra.run.mira028:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:13:13,447.447 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.368211 7f6ac3fc0780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:13,459.459 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.381641 7f6ac3fc0780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:13,461.461 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.382717 7f6ac3fc0780 -1 filestore(/var/lib/ceph/osd/ceph-3) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:13:13,512.512 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.432488 7f6ac3fc0780 -1 created object store /var/lib/ceph/osd/ceph-3 journal /var/lib/ceph/osd/ceph-3/journal for osd.3 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:13,512.512 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.432558 7f6ac3fc0780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2014-07-01 16:13:13,512.512 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.432731 7f6ac3fc0780 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2014-07-01 16:13:13,514.514 INFO:teuthology.orchestra.run.mira028:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:13:13,962.962 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.883640 7f0d73bc9780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:13,972.972 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.893226 7f0d73bc9780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:13,972.972 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.894253 7f0d73bc9780 -1 filestore(/var/lib/ceph/osd/ceph-4) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:13:13,985.985 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.906908 7f0d73bc9780 -1 created object store /var/lib/ceph/osd/ceph-4 journal /var/lib/ceph/osd/ceph-4/journal for osd.4 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:13,986.986 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.906975 7f0d73bc9780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2014-07-01 16:13:13,986.986 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:13.907145 7f0d73bc9780 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2014-07-01 16:13:13,989.989 INFO:teuthology.orchestra.run.mira028:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/monmap' 2014-07-01 16:13:14,507.507 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.428208 7fde584b9780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:14,517.517 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.437770 7fde584b9780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:14,518.518 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.438717 7fde584b9780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-07-01 16:13:14,624.624 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.545874 7fde584b9780 -1 created object store /var/lib/ceph/osd/ceph-5 journal /var/lib/ceph/osd/ceph-5/journal for osd.5 fsid d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:14,625.625 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.545945 7fde584b9780 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2014-07-01 16:13:14,625.625 INFO:teuthology.orchestra.run.mira028.stderr:2014-07-01 07:13:14.546161 7fde584b9780 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2014-07-01 16:13:14,627.627 INFO:teuthology.task.ceph:Reading keys from all nodes... 2014-07-01 16:13:14,628.628 INFO:teuthology.orchestra.run.mira079:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:14,996.996 INFO:teuthology.orchestra.run.mira079:Running: 'sudo cp /var/lib/ceph/mds/ceph-a/keyring /tmp/tmpBBt_cy' 2014-07-01 16:13:15,534.534 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 0666 /tmp/tmpBBt_cy' 2014-07-01 16:13:17,965.965 INFO:teuthology.orchestra.run.mira079:Running: 'rm -fr /tmp/tmpBBt_cy' 2014-07-01 16:13:18,342.342 INFO:teuthology.orchestra.run.mira079:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:18,877.877 INFO:teuthology.orchestra.run.mira079:Running: 'sudo cp /var/lib/ceph/osd/ceph-0/keyring /tmp/tmp_NG3pH' 2014-07-01 16:13:19,468.468 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 0666 /tmp/tmp_NG3pH' 2014-07-01 16:13:21,897.897 INFO:teuthology.orchestra.run.mira079:Running: 'rm -fr /tmp/tmp_NG3pH' 2014-07-01 16:13:22,274.274 INFO:teuthology.orchestra.run.mira079:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:22,811.811 INFO:teuthology.orchestra.run.mira079:Running: 'sudo cp /var/lib/ceph/osd/ceph-1/keyring /tmp/tmpHVrJHB' 2014-07-01 16:13:23,366.366 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 0666 /tmp/tmpHVrJHB' 2014-07-01 16:13:25,795.795 INFO:teuthology.orchestra.run.mira079:Running: 'rm -fr /tmp/tmpHVrJHB' 2014-07-01 16:13:26,173.173 INFO:teuthology.orchestra.run.mira079:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:26,709.709 INFO:teuthology.orchestra.run.mira079:Running: 'sudo cp /var/lib/ceph/osd/ceph-2/keyring /tmp/tmppuBtff' 2014-07-01 16:13:27,264.264 INFO:teuthology.orchestra.run.mira079:Running: 'sudo chmod 0666 /tmp/tmppuBtff' 2014-07-01 16:13:29,696.696 INFO:teuthology.orchestra.run.mira079:Running: 'rm -fr /tmp/tmppuBtff' 2014-07-01 16:13:30,074.074 INFO:teuthology.orchestra.run.mira028:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:30,442.442 INFO:teuthology.orchestra.run.mira028:Running: 'sudo cp /var/lib/ceph/osd/ceph-3/keyring /tmp/tmpWL3QQM' 2014-07-01 16:13:31,018.018 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod 0666 /tmp/tmpWL3QQM' 2014-07-01 16:13:33,569.569 INFO:teuthology.orchestra.run.mira028:Running: 'rm -fr /tmp/tmpWL3QQM' 2014-07-01 16:13:33,963.963 INFO:teuthology.orchestra.run.mira028:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:34,581.581 INFO:teuthology.orchestra.run.mira028:Running: 'sudo cp /var/lib/ceph/osd/ceph-4/keyring /tmp/tmpw34J1g' 2014-07-01 16:13:35,133.133 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod 0666 /tmp/tmpw34J1g' 2014-07-01 16:13:37,683.683 INFO:teuthology.orchestra.run.mira028:Running: 'rm -fr /tmp/tmpw34J1g' 2014-07-01 16:13:38,087.087 INFO:teuthology.orchestra.run.mira028:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-07-01 16:13:38,723.723 INFO:teuthology.orchestra.run.mira028:Running: 'sudo cp /var/lib/ceph/osd/ceph-5/keyring /tmp/tmpJc_6xa' 2014-07-01 16:13:39,338.338 INFO:teuthology.orchestra.run.mira028:Running: 'sudo chmod 0666 /tmp/tmpJc_6xa' 2014-07-01 16:13:41,878.878 INFO:teuthology.orchestra.run.mira028:Running: 'rm -fr /tmp/tmpJc_6xa' 2014-07-01 16:13:43,943.943 INFO:teuthology.task.ceph:Adding keys to all mons... 2014-07-01 16:13:43,944.944 INFO:teuthology.orchestra.run.mira028:Running: 'sudo tee -a /etc/ceph/ceph.keyring' 2014-07-01 16:13:44,313.313 INFO:teuthology.orchestra.run.mira079:Running: 'sudo tee -a /etc/ceph/ceph.keyring' 2014-07-01 16:13:45,130.130 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mds allow --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:45,671.671 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mds allow --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:46,085.085 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:46,457.457 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:46,866.866 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:47,204.204 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:47,580.580 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:47,952.952 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:48,362.362 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:48,734.734 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:49,140.140 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:49,513.513 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:49,923.923 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:50,294.294 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap osd 'allow *' --cap mon 'allow *'" 2014-07-01 16:13:50,703.703 INFO:teuthology.orchestra.run.mira028:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mds allow --cap osd 'allow rwx' --cap mon 'allow rw'" 2014-07-01 16:13:51,077.077 INFO:teuthology.orchestra.run.mira079:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mds allow --cap osd 'allow rwx' --cap mon 'allow rw'" 2014-07-01 16:13:51,484.484 INFO:teuthology.task.ceph:Running mkfs on mon nodes... 2014-07-01 16:13:51,485.485 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-a' 2014-07-01 16:13:51,999.999 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i a --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-07-01 16:13:52,562.562 INFO:teuthology.orchestra.run.mira079.stdout:ceph-mon: set fsid to d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:52,567.567 INFO:teuthology.orchestra.run.mira079.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2014-07-01 16:13:52,572.572 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-b' 2014-07-01 16:13:53,091.091 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i b --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-07-01 16:13:53,664.664 INFO:teuthology.orchestra.run.mira079.stdout:ceph-mon: set fsid to d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:53,670.670 INFO:teuthology.orchestra.run.mira079.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-b for mon.b 2014-07-01 16:13:53,674.674 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-c' 2014-07-01 16:13:54,061.061 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i c --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-07-01 16:13:54,628.628 INFO:teuthology.orchestra.run.mira028.stdout:ceph-mon: set fsid to d0c0ad56-ac30-428a-b414-dd3ce1a5627d 2014-07-01 16:13:54,634.634 INFO:teuthology.orchestra.run.mira028.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-c for mon.c 2014-07-01 16:13:54,639.639 INFO:teuthology.orchestra.run.mira028:Running: 'rm -- /home/ubuntu/cephtest/monmap /home/ubuntu/cephtest/osdmap' 2014-07-01 16:13:55,143.143 INFO:teuthology.orchestra.run.mira079:Running: 'rm -- /home/ubuntu/cephtest/monmap /home/ubuntu/cephtest/osdmap' 2014-07-01 16:13:55,522.522 INFO:teuthology.task.ceph:Starting mon daemons... 2014-07-01 16:13:55,522.522 INFO:teuthology.task.ceph.mon.a:Restarting daemon 2014-07-01 16:13:55,523.523 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i a' 2014-07-01 16:13:56,060.060 INFO:teuthology.task.ceph.mon.a:Started 2014-07-01 16:13:56,061.061 INFO:teuthology.task.ceph.mon.b:Restarting daemon 2014-07-01 16:13:56,061.061 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i b' 2014-07-01 16:13:56,431.431 INFO:teuthology.task.ceph.mon.b:Started 2014-07-01 16:13:56,432.432 INFO:teuthology.task.ceph.mon.c:Restarting daemon 2014-07-01 16:13:56,432.432 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i c' 2014-07-01 16:13:56,766.766 INFO:teuthology.task.ceph.mon.c:Started 2014-07-01 16:13:56,767.767 INFO:teuthology.task.ceph:Starting osd daemons... 2014-07-01 16:13:56,767.767 INFO:teuthology.task.ceph.osd.0:Restarting daemon 2014-07-01 16:13:56,768.768 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 0' 2014-07-01 16:13:57,137.137 INFO:teuthology.task.ceph.osd.0:Started 2014-07-01 16:13:57,137.137 INFO:teuthology.task.ceph.osd.1:Restarting daemon 2014-07-01 16:13:57,137.137 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2014-07-01 16:13:57,199.199 INFO:teuthology.task.ceph.osd.0.mira079.stdout:starting osd.0 at :/0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal 2014-07-01 16:13:57,207.207 INFO:teuthology.task.ceph.osd.0.mira079.stderr:2014-07-01 07:13:57.117788 7fd4e3a69780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:57,471.471 INFO:teuthology.task.ceph.osd.1:Started 2014-07-01 16:13:57,472.472 INFO:teuthology.task.ceph.osd.2:Restarting daemon 2014-07-01 16:13:57,472.472 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 2' 2014-07-01 16:13:57,543.543 INFO:teuthology.task.ceph.osd.1.mira079.stdout:starting osd.1 at :/0 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal 2014-07-01 16:13:57,551.551 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 07:13:57.462172 7ffcfbd0a780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:57,842.842 INFO:teuthology.task.ceph.osd.2:Started 2014-07-01 16:13:57,843.843 INFO:teuthology.task.ceph.osd.3:Restarting daemon 2014-07-01 16:13:57,843.843 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 3' 2014-07-01 16:13:57,909.909 INFO:teuthology.task.ceph.osd.2.mira079.stdout:starting osd.2 at :/0 osd_data /var/lib/ceph/osd/ceph-2 /var/lib/ceph/osd/ceph-2/journal 2014-07-01 16:13:57,915.915 INFO:teuthology.task.ceph.osd.2.mira079.stderr:2014-07-01 07:13:57.824945 7f8d8a282780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:58,213.213 INFO:teuthology.task.ceph.osd.3:Started 2014-07-01 16:13:58,213.213 INFO:teuthology.task.ceph.osd.4:Restarting daemon 2014-07-01 16:13:58,214.214 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 4' 2014-07-01 16:13:58,272.272 INFO:teuthology.task.ceph.osd.3.mira028.stdout:starting osd.3 at :/0 osd_data /var/lib/ceph/osd/ceph-3 /var/lib/ceph/osd/ceph-3/journal 2014-07-01 16:13:58,281.281 INFO:teuthology.task.ceph.osd.3.mira028.stderr:2014-07-01 07:13:58.204003 7fad82f8a780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:58,549.549 INFO:teuthology.task.ceph.osd.4:Started 2014-07-01 16:13:58,549.549 INFO:teuthology.task.ceph.osd.5:Restarting daemon 2014-07-01 16:13:58,550.550 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 5' 2014-07-01 16:13:58,610.610 INFO:teuthology.task.ceph.osd.4.mira028.stdout:starting osd.4 at :/0 osd_data /var/lib/ceph/osd/ceph-4 /var/lib/ceph/osd/ceph-4/journal 2014-07-01 16:13:58,616.616 INFO:teuthology.task.ceph.osd.4.mira028.stderr:2014-07-01 07:13:58.538960 7f4d1e7a5780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:58,920.920 INFO:teuthology.task.ceph.osd.5:Started 2014-07-01 16:13:58,921.921 INFO:teuthology.task.ceph:Starting mds daemons... 2014-07-01 16:13:58,921.921 INFO:teuthology.task.ceph.mds.a:Restarting daemon 2014-07-01 16:13:58,922.922 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f -i a' 2014-07-01 16:13:58,983.983 INFO:teuthology.task.ceph.osd.5.mira028.stdout:starting osd.5 at :/0 osd_data /var/lib/ceph/osd/ceph-5 /var/lib/ceph/osd/ceph-5/journal 2014-07-01 16:13:58,993.993 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:13:58.915912 7fa5e112b780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:13:59,255.255 INFO:teuthology.task.ceph.mds.a:Started 2014-07-01 16:13:59,256.256 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph mds set_max_mds 1' 2014-07-01 16:13:59,323.323 INFO:teuthology.task.ceph.mds.a.mira079.stdout:starting mds.a at :/0 2014-07-01 16:14:00,660.660 INFO:teuthology.orchestra.run.mira079.stderr:max_mds = 1 2014-07-01 16:14:00,673.673 INFO:teuthology.task.ceph:Waiting until ceph is healthy... 2014-07-01 16:14:00,673.673 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:14:01,329.329 DEBUG:teuthology.misc:6 of 6 OSDs are up 2014-07-01 16:14:01,329.329 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-07-01 16:14:02,046.046 DEBUG:teuthology.misc:Ceph health: HEALTH_WARN 72 pgs stuck inactive; 72 pgs stuck unclean 2014-07-01 16:14:09,046.046 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-07-01 16:14:09,658.658 DEBUG:teuthology.misc:Ceph health: HEALTH_OK 2014-07-01 16:14:09,658.658 INFO:teuthology.run_tasks:Running task install.upgrade... 2014-07-01 16:14:09,659.659 INFO:teuthology.task.install:project ceph config {'osd.0': None} overrides {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac'} 2014-07-01 16:14:09,659.659 INFO:teuthology.task.install:extra packages: [] 2014-07-01 16:14:09,659.659 INFO:teuthology.task.install:remote ubuntu@mira079.front.sepia.ceph.com config {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac'} 2014-07-01 16:14:09,660.660 INFO:teuthology.orchestra.run.mira079:Running: 'sudo lsb_release -is' 2014-07-01 16:14:10,356.356 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:14:10,357.357 INFO:teuthology.task.install:Upgrading ceph deb packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, libcephfs-java, librados2, librados2-dbg, librbd1, librbd1-dbg 2014-07-01 16:14:10,357.357 INFO:teuthology.orchestra.run.mira079:Running: 'sudo apt-key list | grep Ceph' 2014-07-01 16:14:11,035.035 INFO:teuthology.orchestra.run.mira079:Running: 'lsb_release -sc' 2014-07-01 16:14:11,644.644 INFO:teuthology.orchestra.run.mira079:Running: 'arch' 2014-07-01 16:14:12,271.271 INFO:teuthology.task.install:dist precise arch x86_64 2014-07-01 16:14:12,271.271 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac 2014-07-01 16:14:12,272.272 INFO:teuthology.orchestra.run.mira079:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac/version' 2014-07-01 16:14:12,970.970 INFO:teuthology.task.install:Package version is 0.81-365-g0aed380-1precise 2014-07-01 16:14:12,971.971 INFO:teuthology.orchestra.run.mira079:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac precise main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-07-01 16:14:13,529.529 INFO:teuthology.orchestra.run.mira079:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.81-365-g0aed380-1precise ceph-dbg=0.81-365-g0aed380-1precise ceph-mds=0.81-365-g0aed380-1precise ceph-mds-dbg=0.81-365-g0aed380-1precise ceph-common=0.81-365-g0aed380-1precise ceph-common-dbg=0.81-365-g0aed380-1precise ceph-fuse=0.81-365-g0aed380-1precise ceph-fuse-dbg=0.81-365-g0aed380-1precise ceph-test=0.81-365-g0aed380-1precise ceph-test-dbg=0.81-365-g0aed380-1precise radosgw=0.81-365-g0aed380-1precise radosgw-dbg=0.81-365-g0aed380-1precise python-ceph=0.81-365-g0aed380-1precise libcephfs1=0.81-365-g0aed380-1precise libcephfs1-dbg=0.81-365-g0aed380-1precise libcephfs-java=0.81-365-g0aed380-1precise librados2=0.81-365-g0aed380-1precise librados2-dbg=0.81-365-g0aed380-1precise librbd1=0.81-365-g0aed380-1precise librbd1-dbg=0.81-365-g0aed380-1precise' 2014-07-01 16:15:11,969.969 INFO:teuthology.orchestra.run.mira079.stderr:start: Job is already running: ceph-all 2014-07-01 16:15:12,019.019 INFO:teuthology.orchestra.run.mira079.stderr:start: Job is already running: radosgw-all 2014-07-01 16:15:12,098.098 INFO:teuthology.orchestra.run.mira079.stderr:start: Job is already running: ceph-mds-all 2014-07-01 16:15:12,328.328 INFO:teuthology.run_tasks:Running task ceph.restart... 2014-07-01 16:15:12,329.329 DEBUG:teuthology.task.ceph.osd.1:waiting for process to exit 2014-07-01 16:15:18,327.327 INFO:teuthology.task.ceph.osd.1:Stopped 2014-07-01 16:15:18,328.328 INFO:teuthology.task.ceph.osd.1:Restarting daemon 2014-07-01 16:15:18,328.328 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2014-07-01 16:15:18,699.699 INFO:teuthology.task.ceph.osd.1:Started 2014-07-01 16:15:18,700.700 DEBUG:teuthology.task.ceph.osd.0:waiting for process to exit 2014-07-01 16:15:18,764.764 INFO:teuthology.task.ceph.osd.1.mira079.stdout:starting osd.1 at :/0 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal 2014-07-01 16:15:18,782.782 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 07:15:18.690852 7f60a7191780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:15:24,700.700 INFO:teuthology.task.ceph.osd.0:Stopped 2014-07-01 16:15:24,701.701 INFO:teuthology.task.ceph.osd.0:Restarting daemon 2014-07-01 16:15:24,701.701 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 0' 2014-07-01 16:15:25,100.100 INFO:teuthology.task.ceph.osd.0:Started 2014-07-01 16:15:25,101.101 DEBUG:teuthology.task.ceph.osd.2:waiting for process to exit 2014-07-01 16:15:25,161.161 INFO:teuthology.task.ceph.osd.0.mira079.stdout:starting osd.0 at :/0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal 2014-07-01 16:15:25,177.177 INFO:teuthology.task.ceph.osd.0.mira079.stderr:2014-07-01 07:15:25.085742 7f3d3ee7e780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:15:31,100.100 INFO:teuthology.task.ceph.osd.2:Stopped 2014-07-01 16:15:31,101.101 INFO:teuthology.task.ceph.osd.2:Restarting daemon 2014-07-01 16:15:31,101.101 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 2' 2014-07-01 16:15:31,475.475 INFO:teuthology.task.ceph.osd.2:Started 2014-07-01 16:15:31,475.475 INFO:teuthology.task.ceph:Waiting until ceph is healthy... 2014-07-01 16:15:31,475.475 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:15:31,530.530 INFO:teuthology.task.ceph.osd.2.mira079.stdout:starting osd.2 at :/0 osd_data /var/lib/ceph/osd/ceph-2 /var/lib/ceph/osd/ceph-2/journal 2014-07-01 16:15:31,546.546 INFO:teuthology.task.ceph.osd.2.mira079.stderr:2014-07-01 07:15:31.454477 7fe492c66780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:15:32,029.029 DEBUG:teuthology.misc:6 of 6 OSDs are up 2014-07-01 16:15:32,030.030 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-07-01 16:15:32,770.770 DEBUG:teuthology.misc:Ceph health: HEALTH_WARN 7 pgs peering; 10 pgs stale; 1/6 in osds are down 2014-07-01 16:15:39,771.771 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-07-01 16:15:40,348.348 DEBUG:teuthology.misc:Ceph health: HEALTH_OK 2014-07-01 16:15:40,348.348 INFO:teuthology.run_tasks:Running task thrashosds... 2014-07-01 16:15:40,349.349 INFO:teuthology.task.thrashosds:Beginning thrashosds... 2014-07-01 16:15:40,350.350 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:15:41,098.098 INFO:teuthology.task.thrashosds.ceph_manager:[{u'cache_target_full_ratio_micro': 0, u'stripe_width': 0, u'flags_names': u'hashpspool', u'tier_of': -1, u'pg_placement_num': 24, u'quota_max_bytes': 0, u'erasure_code_profile': u'', u'size': 2, u'snap_seq': 0, u'auid': 0, u'cache_min_flush_age': 0, u'hit_set_period': 0, u'target_max_objects': 0, u'pg_num': 24, u'type': 1, u'crush_ruleset': 0, u'pool_name': u'data', u'cache_min_evict_age': 0, u'snap_mode': u'selfmanaged', u'cache_mode': u'none', u'min_size': 1, u'crash_replay_interval': 45, u'object_hash': 2, u'write_tier': -1, u'cache_target_dirty_ratio_micro': 0, u'pool': 0, u'removed_snaps': u'[]', u'tiers': [], u'hit_set_params': {u'type': u'none'}, u'last_force_op_resend': u'0', u'pool_snaps': [], u'quota_max_objects': 0, u'hit_set_count': 0, u'flags': 1, u'target_max_bytes': 0, u'snap_epoch': 0, u'last_change': u'1', u'read_tier': -1}, {u'cache_target_full_ratio_micro': 0, u'stripe_width': 0, u'flags_names': u'hashpspool', u'tier_of': -1, u'pg_placement_num': 24, u'quota_max_bytes': 0, u'erasure_code_profile': u'', u'size': 2, u'snap_seq': 0, u'auid': 0, u'cache_min_flush_age': 0, u'hit_set_period': 0, u'target_max_objects': 0, u'pg_num': 24, u'type': 1, u'crush_ruleset': 0, u'pool_name': u'metadata', u'cache_min_evict_age': 0, u'snap_mode': u'selfmanaged', u'cache_mode': u'none', u'min_size': 1, u'crash_replay_interval': 0, u'object_hash': 2, u'write_tier': -1, u'cache_target_dirty_ratio_micro': 0, u'pool': 1, u'removed_snaps': u'[]', u'tiers': [], u'hit_set_params': {u'type': u'none'}, u'last_force_op_resend': u'0', u'pool_snaps': [], u'quota_max_objects': 0, u'hit_set_count': 0, u'flags': 1, u'target_max_bytes': 0, u'snap_epoch': 0, u'last_change': u'1', u'read_tier': -1}, {u'cache_target_full_ratio_micro': 0, u'stripe_width': 0, u'flags_names': u'hashpspool', u'tier_of': -1, u'pg_placement_num': 24, u'quota_max_bytes': 0, u'erasure_code_profile': u'', u'size': 2, u'snap_seq': 0, u'auid': 0, u'cache_min_flush_age': 0, u'hit_set_period': 0, u'target_max_objects': 0, u'pg_num': 24, u'type': 1, u'crush_ruleset': 0, u'pool_name': u'rbd', u'cache_min_evict_age': 0, u'snap_mode': u'selfmanaged', u'cache_mode': u'none', u'min_size': 1, u'crash_replay_interval': 0, u'object_hash': 2, u'write_tier': -1, u'cache_target_dirty_ratio_micro': 0, u'pool': 2, u'removed_snaps': u'[]', u'tiers': [], u'hit_set_params': {u'type': u'none'}, u'last_force_op_resend': u'0', u'pool_snaps': [], u'quota_max_objects': 0, u'hit_set_count': 0, u'flags': 1, u'target_max_bytes': 0, u'snap_epoch': 0, u'last_change': u'1', u'read_tier': -1}] 2014-07-01 16:15:41,099.099 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool get data pg_num' 2014-07-01 16:15:41,860.860 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool get metadata pg_num' 2014-07-01 16:15:42,582.582 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool get rbd pg_num' 2014-07-01 16:15:43,331.331 INFO:teuthology.task.thrashosds.ceph_manager:waiting for clean 2014-07-01 16:15:43,331.331 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:15:44,259.259 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:15:44,621.621 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:15:45,405.405 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:15:45,762.762 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:15:46,523.523 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003786', u'skew': u'-0.010307', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002468', u'skew': u'-0.010296', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:14:56.755772', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884794884, u'health': u'HEALTH_OK', u'kb_used': 11986168, u'store_stats': {u'bytes_total': 526185, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}, {u'last_updated': u'2014-07-01 07:14:56.033832', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887347876, u'health': u'HEALTH_OK', u'kb_used': 9433176, u'store_stats': {u'bytes_total': 1050473, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}, {u'last_updated': u'2014-07-01 07:14:56.404627', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887305736, u'health': u'HEALTH_OK', u'kb_used': 9475316, u'store_stats': {u'bytes_total': 1050473, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}]}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 72, u'data_bytes': 1884, u'bytes_used': 838156288, u'version': 27, u'pgs_by_state': [{u'count': 3, u'state_name': u'active+clean+replay'}, {u'count': 69, u'state_name': u'active+clean'}], u'bytes_avail': 5997460889600}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 15, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:15:46,524.524 INFO:teuthology.task.thrashosds.ceph_manager:clean! 2014-07-01 16:15:46,525.525 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump' 2014-07-01 16:15:47,279.279 INFO:teuthology.task.thrashosds.ceph_manager:['osd.0 up in weight 1 up_from 11 up_thru 14 down_at 10 last_clean_interval [5,6) 10.214.135.128:6800/6778 10.214.135.128:6804/6778 10.214.135.128:6820/6778 10.214.135.128:6821/6778 exists,up 6c5f8617-744a-4c20-95c2-5d17e79237af', 'osd.1 up in weight 1 up_from 8 up_thru 14 down_at 7 last_clean_interval [2,6) 10.214.135.128:6809/6651 10.214.135.128:6816/6651 10.214.135.128:6817/6651 10.214.135.128:6818/6651 exists,up 2c69eaff-f1b3-478a-8aef-0dc5cb5eba12', 'osd.2 up in weight 1 up_from 14 up_thru 14 down_at 13 last_clean_interval [3,9) 10.214.135.128:6814/6911 10.214.135.128:6823/6911 10.214.135.128:6824/6911 10.214.135.128:6825/6911 exists,up 7cd8a808-dbc5-4aeb-8348-d282d695437b', 'osd.3 up in weight 1 up_from 3 up_thru 14 down_at 0 last_clean_interval [0,0) 10.214.132.120:6800/6350 10.214.132.120:6801/6350 10.214.132.120:6802/6350 10.214.132.120:6803/6350 exists,up 88a5f50d-1cee-44b5-af17-1355a29fdb93', 'osd.4 up in weight 1 up_from 4 up_thru 14 down_at 0 last_clean_interval [0,0) 10.214.132.120:6805/6415 10.214.132.120:6806/6415 10.214.132.120:6807/6415 10.214.132.120:6808/6415 exists,up 865d87c5-d521-40fa-9005-24145fc4561e', 'osd.5 up in weight 1 up_from 4 up_thru 14 down_at 0 last_clean_interval [0,0) 10.214.132.120:6810/6502 10.214.132.120:6811/6502 10.214.132.120:6812/6502 10.214.132.120:6813/6502 exists,up 7ab5f2a8-a2dc-447d-af65-a80ae1767e11'] 2014-07-01 16:15:47,280.280 INFO:teuthology.orchestra.run.mira079:Running: "adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- tell 'mon.*' injectargs '--mon-osd-down-out-interval 0'" 2014-07-01 16:15:47,990.990 INFO:teuthology.orchestra.run.mira079.stderr:mon.c: injectargs:mon_osd_down_out_interval = '0' 2014-07-01 16:15:48,079.079 INFO:teuthology.orchestra.run.mira079.stderr:mon.a: injectargs:mon_osd_down_out_interval = '0' 2014-07-01 16:15:48,159.159 INFO:teuthology.orchestra.run.mira079.stderr:mon.b: injectargs:mon_osd_down_out_interval = '0' 2014-07-01 16:15:48,172.172 INFO:teuthology.run_tasks:Running task ceph.restart... 2014-07-01 16:15:48,172.172 DEBUG:teuthology.task.ceph.mon.a:waiting for process to exit 2014-07-01 16:15:48,172.172 INFO:teuthology.task.thrashosds.thrasher:starting do_thrash 2014-07-01 16:15:48,172.172 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:15:48,172.172 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:15:48,172.172 INFO:teuthology.task.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5] 2014-07-01 16:15:48,173.173 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 2' 2014-07-01 16:15:54,173.173 INFO:teuthology.task.ceph.mon.a:Stopped 2014-07-01 16:15:54,174.174 INFO:teuthology.task.ceph.mon.a:Restarting daemon 2014-07-01 16:15:54,174.174 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i a' 2014-07-01 16:15:54,544.544 INFO:teuthology.task.ceph.mon.a:Started 2014-07-01 16:15:54,544.544 INFO:teuthology.task.ceph:Waiting until ceph osds are all up... 2014-07-01 16:15:54,544.544 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:15:54,648.648 INFO:teuthology.orchestra.run.mira079.stderr:osd.2 is already out. 2014-07-01 16:15:55,298.298 DEBUG:teuthology.misc:6 of 6 OSDs are up 2014-07-01 16:15:55,298.298 INFO:teuthology.run_tasks:Running task rados... 2014-07-01 16:15:55,299.299 INFO:teuthology.task.rados:Beginning rados... 2014-07-01 16:15:55,299.299 INFO:teuthology.run_tasks:Running task ceph.restart... 2014-07-01 16:15:55,300.300 DEBUG:teuthology.task.ceph.mon.b:waiting for process to exit 2014-07-01 16:15:55,300.300 INFO:teuthology.task.rados:clients are ['client.0'] 2014-07-01 16:15:55,300.300 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 ruleset-failure-domain=osd' 2014-07-01 16:15:59,662.662 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:15:59,662.662 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:16:01,297.297 INFO:teuthology.task.ceph.mon.b:Stopped 2014-07-01 16:16:01,297.297 INFO:teuthology.task.ceph.mon.b:Restarting daemon 2014-07-01 16:16:01,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i b' 2014-07-01 16:16:01,670.670 INFO:teuthology.task.ceph.mon.b:Started 2014-07-01 16:16:01,670.670 INFO:teuthology.task.ceph:Waiting until ceph osds are all up... 2014-07-01 16:16:01,670.670 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:16:02,219.219 DEBUG:teuthology.misc:6 of 6 OSDs are up 2014-07-01 16:16:02,220.220 INFO:teuthology.run_tasks:Running task radosbench... 2014-07-01 16:16:02,220.220 INFO:teuthology.task.radosbench:Beginning radosbench... 2014-07-01 16:16:02,826.826 INFO:teuthology.task.rados:starting run 0 out of 1 2014-07-01 16:16:02,827.827 INFO:teuthology.task.thrashosds.ceph_manager:creating pool_name unique_pool_0 2014-07-01 16:16:02,827.827 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile' 2014-07-01 16:16:03,798.798 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:16:03,813.813 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:16:04,802.802 INFO:teuthology.orchestra.run.mira079.stderr:pool 'unique_pool_0' created 2014-07-01 16:16:04,816.816 INFO:teuthology.orchestra.run.mira043:Running: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --ec-pool --op read 45 --op write 0 --op delete 10 --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op append 45 --pool unique_pool_0' 2014-07-01 16:16:04,816.816 INFO:teuthology.task.thrashosds.ceph_manager:creating pool_name unique_pool_1 2014-07-01 16:16:04,816.816 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool create unique_pool_1 16' 2014-07-01 16:16:04,830.830 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:16:04,845.845 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:16:05,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight read -> 45 2014-07-01 16:16:05,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight delete -> 10 2014-07-01 16:16:05,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight append -> 45 2014-07-01 16:16:05,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:ceph version 0.80.1-50-gd43e711 (d43e7113dd501aea1db33fdae30d56e96e9c3897) 2014-07-01 16:16:05,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:Configuration: 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Number of operations: 4000 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Number of objects: 500 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Max in flight operations: 16 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Object size (in bytes): 4000000 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Write stride min: 400000 2014-07-01 16:16:05,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout: Write stride max: 800000 2014-07-01 16:16:05,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: write initial oid 1 2014-07-01 16:16:05,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: seq_num 0 ranges {0=614400,614400=401408} 2014-07-01 16:16:05,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: writing mira0435601-1 from 0 to 614400 tid 1 2014-07-01 16:16:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: writing mira0435601-1 from 614400 to 1015808 tid 2 2014-07-01 16:16:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: write initial oid 2 2014-07-01 16:16:05,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: seq_num 1 ranges {0=638976,638976=520192,1159168=700416,1859584=385024} 2014-07-01 16:16:05,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: writing mira0435601-2 from 0 to 638976 tid 1 2014-07-01 16:16:05,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: writing mira0435601-2 from 638976 to 1159168 tid 2 2014-07-01 16:16:05,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: writing mira0435601-2 from 1159168 to 1859584 tid 3 2014-07-01 16:16:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: writing mira0435601-2 from 1859584 to 2244608 tid 4 2014-07-01 16:16:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: write initial oid 3 2014-07-01 16:16:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: seq_num 2 ranges {0=667648,667648=401408} 2014-07-01 16:16:05,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: writing mira0435601-3 from 0 to 667648 tid 1 2014-07-01 16:16:05,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: writing mira0435601-3 from 667648 to 1069056 tid 2 2014-07-01 16:16:05,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: write initial oid 4 2014-07-01 16:16:05,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: seq_num 3 ranges {0=782336,782336=737280,1519616=466944} 2014-07-01 16:16:05,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: writing mira0435601-4 from 0 to 782336 tid 1 2014-07-01 16:16:05,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: writing mira0435601-4 from 782336 to 1519616 tid 2 2014-07-01 16:16:05,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: writing mira0435601-4 from 1519616 to 1986560 tid 3 2014-07-01 16:16:05,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: write initial oid 5 2014-07-01 16:16:05,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: seq_num 4 ranges {0=409600,409600=401408} 2014-07-01 16:16:05,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: writing mira0435601-5 from 0 to 409600 tid 1 2014-07-01 16:16:05,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: writing mira0435601-5 from 409600 to 811008 tid 2 2014-07-01 16:16:05,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: write initial oid 6 2014-07-01 16:16:05,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: seq_num 5 ranges {0=487424} 2014-07-01 16:16:05,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: writing mira0435601-6 from 0 to 487424 tid 1 2014-07-01 16:16:05,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: write initial oid 7 2014-07-01 16:16:05,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: seq_num 6 ranges {0=602112,602112=577536,1179648=225280} 2014-07-01 16:16:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0435601-7 from 0 to 602112 tid 1 2014-07-01 16:16:05,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0435601-7 from 602112 to 1179648 tid 2 2014-07-01 16:16:05,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0435601-7 from 1179648 to 1404928 tid 3 2014-07-01 16:16:05,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: write initial oid 8 2014-07-01 16:16:05,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: seq_num 7 ranges {0=229376} 2014-07-01 16:16:05,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: writing mira0435601-8 from 0 to 229376 tid 1 2014-07-01 16:16:05,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: write initial oid 9 2014-07-01 16:16:05,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: seq_num 8 ranges {0=659456,659456=745472,1404928=57344} 2014-07-01 16:16:05,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0435601-9 from 0 to 659456 tid 1 2014-07-01 16:16:05,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0435601-9 from 659456 to 1404928 tid 2 2014-07-01 16:16:05,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0435601-9 from 1404928 to 1462272 tid 3 2014-07-01 16:16:05,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: write initial oid 10 2014-07-01 16:16:05,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: seq_num 9 ranges {0=528384,528384=729088,1257472=73728} 2014-07-01 16:16:05,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: writing mira0435601-10 from 0 to 528384 tid 1 2014-07-01 16:16:05,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: writing mira0435601-10 from 528384 to 1257472 tid 2 2014-07-01 16:16:05,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: writing mira0435601-10 from 1257472 to 1331200 tid 3 2014-07-01 16:16:05,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: write initial oid 11 2014-07-01 16:16:05,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: seq_num 10 ranges {0=450560,450560=557056,1007616=667648,1675264=380928} 2014-07-01 16:16:05,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0435601-11 from 0 to 450560 tid 1 2014-07-01 16:16:05,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0435601-11 from 450560 to 1007616 tid 2 2014-07-01 16:16:05,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0435601-11 from 1007616 to 1675264 tid 3 2014-07-01 16:16:05,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0435601-11 from 1675264 to 2056192 tid 4 2014-07-01 16:16:05,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: write initial oid 12 2014-07-01 16:16:05,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: seq_num 11 ranges {0=479232,479232=401408} 2014-07-01 16:16:05,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: writing mira0435601-12 from 0 to 479232 tid 1 2014-07-01 16:16:05,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: writing mira0435601-12 from 479232 to 880640 tid 2 2014-07-01 16:16:05,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: write initial oid 13 2014-07-01 16:16:05,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: seq_num 12 ranges {0=749568} 2014-07-01 16:16:05,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: writing mira0435601-13 from 0 to 749568 tid 1 2014-07-01 16:16:05,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: write initial oid 14 2014-07-01 16:16:05,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: seq_num 13 ranges {0=622592} 2014-07-01 16:16:05,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: writing mira0435601-14 from 0 to 622592 tid 1 2014-07-01 16:16:05,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: write initial oid 15 2014-07-01 16:16:05,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: seq_num 14 ranges {0=299008} 2014-07-01 16:16:05,836.836 INFO:teuthology.orchestra.run.mira079.stderr:pool 'unique_pool_1' created 2014-07-01 16:16:05,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: writing mira0435601-15 from 0 to 299008 tid 1 2014-07-01 16:16:05,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: write initial oid 16 2014-07-01 16:16:05,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: seq_num 15 ranges {0=167936} 2014-07-01 16:16:05,841.841 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:16:05,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: writing mira0435601-16 from 0 to 167936 tid 1 2014-07-01 16:16:05,848.848 INFO:teuthology.orchestra.run.mira043:Running: "/bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --name client.0 -p unique_pool_1 bench 1800 write'" 2014-07-01 16:16:05,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:05,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 1 to mira0435601-7 2014-07-01 16:16:05,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 2 to mira0435601-7 2014-07-01 16:16:05,856.856 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:16:05,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 3 to mira0435601-7 2014-07-01 16:16:05,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 1 to mira0435601-1 2014-07-01 16:16:05,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 5 to mira0435601-7 2014-07-01 16:16:05,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 1 to mira0435601-3 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 6 to mira0435601-7 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 7 v 4 (ObjNum 6 snap 0 seq_num 6) dirty exists 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 1 to mira0435601-6 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: done (15 left) 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: write initial oid 17 2014-07-01 16:16:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: seq_num 16 ranges {0=40960} 2014-07-01 16:16:05,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: writing mira0435601-17 from 0 to 40960 tid 1 2014-07-01 16:16:05,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:05,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 2 to mira0435601-1 2014-07-01 16:16:05,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 2 to mira0435601-3 2014-07-01 16:16:05,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 3 to mira0435601-6 2014-07-01 16:16:05,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 1 to mira0435601-2 2014-07-01 16:16:05,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 1 to mira0435601-10 2014-07-01 16:16:05,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 4 to mira0435601-3 2014-07-01 16:16:05,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 2 to mira0435601-2 2014-07-01 16:16:05,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 2 to mira0435601-10 2014-07-01 16:16:05,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 3 to mira0435601-10 2014-07-01 16:16:05,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 5 to mira0435601-10 2014-07-01 16:16:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 1 to mira0435601-5 2014-07-01 16:16:05,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 1 to mira0435601-17 2014-07-01 16:16:05,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 3 to mira0435601-2 2014-07-01 16:16:05,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 4 to mira0435601-2 2014-07-01 16:16:05,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 6 to mira0435601-2 2014-07-01 16:16:05,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 1 to mira0435601-8 2014-07-01 16:16:05,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 4 to mira0435601-6 2014-07-01 16:16:05,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2014-07-01 16:16:05,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: done (15 left) 2014-07-01 16:16:05,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: write initial oid 18 2014-07-01 16:16:05,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: seq_num 17 ranges {0=471040,471040=536576,1007616=266240} 2014-07-01 16:16:05,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0435601-18 from 0 to 471040 tid 1 2014-07-01 16:16:05,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0435601-18 from 471040 to 1007616 tid 2 2014-07-01 16:16:05,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0435601-18 from 1007616 to 1273856 tid 3 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 1 to mira0435601-12 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 2 to mira0435601-12 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 2 to mira0435601-5 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 3 to mira0435601-17 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 5 to mira0435601-3 2014-07-01 16:16:05,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 4 to mira0435601-5 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 4 to mira0435601-12 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 4 to mira0435601-1 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 1 to mira0435601-13 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 3 to mira0435601-8 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 1 to mira0435601-9 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 3 to mira0435601-13 2014-07-01 16:16:05,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 1 to mira0435601-16 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 2 to mira0435601-9 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 7 to mira0435601-2 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 2 v 5 (ObjNum 1 snap 0 seq_num 1) dirty exists 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: done (15 left) 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: done (14 left) 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: write initial oid 19 2014-07-01 16:16:05,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: seq_num 18 ranges {0=741376,741376=401408} 2014-07-01 16:16:05,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: writing mira0435601-19 from 0 to 741376 tid 1 2014-07-01 16:16:05,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: writing mira0435601-19 from 741376 to 1142784 tid 2 2014-07-01 16:16:05,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: write initial oid 20 2014-07-01 16:16:05,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: seq_num 19 ranges {0=663552,663552=749568,1413120=434176,1847296=20480} 2014-07-01 16:16:06,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0435601-20 from 0 to 663552 tid 1 2014-07-01 16:16:06,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0435601-20 from 663552 to 1413120 tid 2 2014-07-01 16:16:06,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0435601-20 from 1413120 to 1847296 tid 3 2014-07-01 16:16:06,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0435601-20 from 1847296 to 1867776 tid 4 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 3 to mira0435601-9 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 6 to mira0435601-10 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 10 v 4 (ObjNum 9 snap 0 seq_num 9) dirty exists 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 4 to mira0435601-8 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 8 v 7 (ObjNum 7 snap 0 seq_num 7) dirty exists 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 4 to mira0435601-17 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 17 v 2 (ObjNum 16 snap 0 seq_num 16) dirty exists 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 5 to mira0435601-5 2014-07-01 16:16:06,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 3 (ObjNum 4 snap 0 seq_num 4) dirty exists 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 1 to mira0435601-4 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 5 to mira0435601-9 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 3 to mira0435601-16 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 2 to mira0435601-4 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 3 to mira0435601-4 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 6 to mira0435601-9 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 4 (ObjNum 8 snap 0 seq_num 8) dirty exists 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 4 to mira0435601-13 2014-07-01 16:16:06,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 5 (ObjNum 12 snap 0 seq_num 12) dirty exists 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 1 to mira0435601-18 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 4 to mira0435601-16 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 7 (ObjNum 15 snap 0 seq_num 15) dirty exists 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 5 to mira0435601-12 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 7 (ObjNum 11 snap 0 seq_num 11) dirty exists 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 5 to mira0435601-4 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 5 to mira0435601-1 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 2 to mira0435601-18 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 6 to mira0435601-4 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 7 (ObjNum 3 snap 0 seq_num 3) dirty exists 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 1 to mira0435601-11 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 2 to mira0435601-11 2014-07-01 16:16:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 3 to mira0435601-18 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 5 to mira0435601-18 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 6 to mira0435601-18 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 18 v 7 (ObjNum 17 snap 0 seq_num 17) dirty exists 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 3 to mira0435601-11 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 4 to mira0435601-11 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 6 to mira0435601-11 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 1 to mira0435601-14 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 3 to mira0435601-14 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 1 to mira0435601-15 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 3 to mira0435601-15 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 1 to mira0435601-20 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 7 to mira0435601-11 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 12 (ObjNum 10 snap 0 seq_num 10) dirty exists 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 4 to mira0435601-14 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 4 to mira0435601-15 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 1 to mira0435601-19 2014-07-01 16:16:06,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: done (15 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: done (14 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: done (13 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: done (12 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: done (11 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: done (10 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: done (9 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: done (8 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: done (7 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: done (6 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: done (5 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: done (4 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: done (3 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: done (2 left) 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: write initial oid 21 2014-07-01 16:16:06,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: seq_num 20 ranges {0=692224} 2014-07-01 16:16:06,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: writing mira0435601-21 from 0 to 692224 tid 1 2014-07-01 16:16:06,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: write initial oid 22 2014-07-01 16:16:06,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: seq_num 21 ranges {0=561152} 2014-07-01 16:16:06,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: writing mira0435601-22 from 0 to 561152 tid 1 2014-07-01 16:16:06,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: write initial oid 23 2014-07-01 16:16:06,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: seq_num 22 ranges {0=589824,589824=745472,1335296=458752} 2014-07-01 16:16:06,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: writing mira0435601-23 from 0 to 589824 tid 1 2014-07-01 16:16:06,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: writing mira0435601-23 from 589824 to 1335296 tid 2 2014-07-01 16:16:06,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: writing mira0435601-23 from 1335296 to 1794048 tid 3 2014-07-01 16:16:06,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: write initial oid 24 2014-07-01 16:16:06,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: seq_num 23 ranges {0=106496} 2014-07-01 16:16:06,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: writing mira0435601-24 from 0 to 106496 tid 1 2014-07-01 16:16:06,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: write initial oid 25 2014-07-01 16:16:06,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: seq_num 24 ranges {0=782336,782336=512000,1294336=552960,1847296=540672} 2014-07-01 16:16:06,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: writing mira0435601-25 from 0 to 782336 tid 1 2014-07-01 16:16:06,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: writing mira0435601-25 from 782336 to 1294336 tid 2 2014-07-01 16:16:06,182.182 INFO:teuthology.run_tasks:Running task install.upgrade... 2014-07-01 16:16:06,182.182 INFO:teuthology.task.install:project ceph config {'mon.c': None} overrides {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac'} 2014-07-01 16:16:06,183.183 INFO:teuthology.task.install:extra packages: [] 2014-07-01 16:16:06,183.183 INFO:teuthology.task.install:remote ubuntu@mira028.front.sepia.ceph.com config {'sha1': '0aed380a61114710cf5edf49472d1c91e31991ac'} 2014-07-01 16:16:06,183.183 INFO:teuthology.orchestra.run.mira028:Running: 'sudo lsb_release -is' 2014-07-01 16:16:06,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: writing mira0435601-25 from 1294336 to 1847296 tid 3 2014-07-01 16:16:06,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: writing mira0435601-25 from 1847296 to 2387968 tid 4 2014-07-01 16:16:06,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: write initial oid 26 2014-07-01 16:16:06,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: seq_num 25 ranges {0=409600,409600=495616,905216=307200} 2014-07-01 16:16:06,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0435601-26 from 0 to 409600 tid 1 2014-07-01 16:16:06,214.214 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: Maintaining 16 concurrent writes of 4194304 bytes for up to 1800 seconds or 0 objects 2014-07-01 16:16:06,214.214 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: Object prefix: benchmark_data_mira043_5625 2014-07-01 16:16:06,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0435601-26 from 409600 to 905216 tid 2 2014-07-01 16:16:06,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0435601-26 from 905216 to 1212416 tid 3 2014-07-01 16:16:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: write initial oid 27 2014-07-01 16:16:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: seq_num 26 ranges {0=679936,679936=401408} 2014-07-01 16:16:06,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: writing mira0435601-27 from 0 to 679936 tid 1 2014-07-01 16:16:06,260.260 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:16:06,261.261 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 0 0 0 0 0 0 - 0 2014-07-01 16:16:06,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: writing mira0435601-27 from 679936 to 1081344 tid 2 2014-07-01 16:16:06,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: write initial oid 28 2014-07-01 16:16:06,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: seq_num 27 ranges {0=552960,552960=401408} 2014-07-01 16:16:06,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: writing mira0435601-28 from 0 to 552960 tid 1 2014-07-01 16:16:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: writing mira0435601-28 from 552960 to 954368 tid 2 2014-07-01 16:16:06,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: write initial oid 29 2014-07-01 16:16:06,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: seq_num 28 ranges {0=630784} 2014-07-01 16:16:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: writing mira0435601-29 from 0 to 630784 tid 1 2014-07-01 16:16:06,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: write initial oid 30 2014-07-01 16:16:06,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: seq_num 29 ranges {0=499712} 2014-07-01 16:16:06,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: writing mira0435601-30 from 0 to 499712 tid 1 2014-07-01 16:16:06,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: write initial oid 31 2014-07-01 16:16:06,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: seq_num 30 ranges {0=372736} 2014-07-01 16:16:06,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: writing mira0435601-31 from 0 to 372736 tid 1 2014-07-01 16:16:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: write initial oid 32 2014-07-01 16:16:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: seq_num 31 ranges {0=802816,802816=536576,1339392=266240} 2014-07-01 16:16:06,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0435601-32 from 0 to 802816 tid 1 2014-07-01 16:16:06,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0435601-32 from 802816 to 1339392 tid 2 2014-07-01 16:16:06,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0435601-32 from 1339392 to 1605632 tid 3 2014-07-01 16:16:06,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: write initial oid 33 2014-07-01 16:16:06,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: seq_num 32 ranges {0=475136,475136=471040,946176=331776} 2014-07-01 16:16:06,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0435601-33 from 0 to 475136 tid 1 2014-07-01 16:16:06,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0435601-33 from 475136 to 946176 tid 2 2014-07-01 16:16:06,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0435601-33 from 946176 to 1277952 tid 3 2014-07-01 16:16:06,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: write initial oid 34 2014-07-01 16:16:06,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: seq_num 33 ranges {0=593920,593920=704512,1298432=720896,2019328=180224} 2014-07-01 16:16:06,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: writing mira0435601-34 from 0 to 593920 tid 1 2014-07-01 16:16:06,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: writing mira0435601-34 from 593920 to 1298432 tid 2 2014-07-01 16:16:06,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: writing mira0435601-34 from 1298432 to 2019328 tid 3 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: writing mira0435601-34 from 2019328 to 2199552 tid 4 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 2 to mira0435601-19 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 4 to mira0435601-19 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 5 to mira0435601-19 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 15 (ObjNum 18 snap 0 seq_num 18) dirty exists 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 2 to mira0435601-20 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 3 to mira0435601-20 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 4 to mira0435601-20 2014-07-01 16:16:06,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 6 to mira0435601-20 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 7 to mira0435601-20 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 9 (ObjNum 19 snap 0 seq_num 19) dirty exists 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 1 to mira0435601-21 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 3 to mira0435601-21 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 4 to mira0435601-21 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 11 (ObjNum 20 snap 0 seq_num 20) dirty exists 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 1 to mira0435601-22 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 3 to mira0435601-22 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 4 to mira0435601-22 2014-07-01 16:16:06,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 22 v 9 (ObjNum 21 snap 0 seq_num 21) dirty exists 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 1 to mira0435601-23 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 2 to mira0435601-23 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 1 to mira0435601-24 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 3 to mira0435601-24 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 3 to mira0435601-23 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 5 to mira0435601-23 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 4 to mira0435601-24 2014-07-01 16:16:06,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 6 to mira0435601-23 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 1 to mira0435601-25 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 2 to mira0435601-25 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 3 to mira0435601-25 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 4 to mira0435601-25 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 6 to mira0435601-25 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 1 to mira0435601-26 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 2 to mira0435601-26 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 3 to mira0435601-26 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 5 to mira0435601-26 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 1 to mira0435601-27 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 2 to mira0435601-27 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: done (15 left) 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: done (14 left) 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: done (13 left) 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: done (12 left) 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: done (11 left) 2014-07-01 16:16:06,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: done (10 left) 2014-07-01 16:16:06,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: write initial oid 35 2014-07-01 16:16:06,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: seq_num 34 ranges {0=622592,622592=401408} 2014-07-01 16:16:06,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: writing mira0435601-35 from 0 to 622592 tid 1 2014-07-01 16:16:06,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: writing mira0435601-35 from 622592 to 1024000 tid 2 2014-07-01 16:16:06,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: write initial oid 36 2014-07-01 16:16:06,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: seq_num 35 ranges {0=647168,647168=516096,1163264=774144,1937408=315392} 2014-07-01 16:16:06,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0435601-36 from 0 to 647168 tid 1 2014-07-01 16:16:06,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0435601-36 from 647168 to 1163264 tid 2 2014-07-01 16:16:06,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0435601-36 from 1163264 to 1937408 tid 3 2014-07-01 16:16:06,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0435601-36 from 1937408 to 2252800 tid 4 2014-07-01 16:16:06,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: write initial oid 37 2014-07-01 16:16:06,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: seq_num 36 ranges {0=765952} 2014-07-01 16:16:06,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: writing mira0435601-37 from 0 to 765952 tid 1 2014-07-01 16:16:06,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: write initial oid 38 2014-07-01 16:16:06,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: seq_num 37 ranges {0=442368} 2014-07-01 16:16:06,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: writing mira0435601-38 from 0 to 442368 tid 1 2014-07-01 16:16:06,619.619 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:16:06,619.619 INFO:teuthology.task.install:Upgrading ceph deb packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, libcephfs-java, librados2, librados2-dbg, librbd1, librbd1-dbg 2014-07-01 16:16:06,619.619 INFO:teuthology.orchestra.run.mira028:Running: 'sudo apt-key list | grep Ceph' 2014-07-01 16:16:06,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: write initial oid 39 2014-07-01 16:16:06,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: seq_num 38 ranges {0=311296} 2014-07-01 16:16:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: writing mira0435601-39 from 0 to 311296 tid 1 2014-07-01 16:16:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: write initial oid 40 2014-07-01 16:16:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: seq_num 39 ranges {0=184320} 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: writing mira0435601-40 from 0 to 184320 tid 1 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 4 to mira0435601-27 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 5 to mira0435601-27 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 18 (ObjNum 26 snap 0 seq_num 26) dirty exists 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 1 to mira0435601-28 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 2 to mira0435601-28 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 4 to mira0435601-28 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: done (15 left) 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: write initial oid 41 2014-07-01 16:16:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: seq_num 40 ranges {0=614400,614400=679936,1294336=122880} 2014-07-01 16:16:06,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0435601-41 from 0 to 614400 tid 1 2014-07-01 16:16:06,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0435601-41 from 614400 to 1294336 tid 2 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0435601-41 from 1294336 to 1417216 tid 3 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 1 to mira0435601-30 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 3 to mira0435601-30 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 1 to mira0435601-31 2014-07-01 16:16:06,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 3 to mira0435601-31 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 1 to mira0435601-29 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 3 to mira0435601-29 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 4 to mira0435601-29 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 9 (ObjNum 28 snap 0 seq_num 28) dirty exists 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 1 to mira0435601-32 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 2 to mira0435601-32 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 3 to mira0435601-32 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 5 to mira0435601-32 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 1 to mira0435601-34 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 1 to mira0435601-33 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 2 to mira0435601-34 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 3 to mira0435601-34 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 4 to mira0435601-34 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 6 to mira0435601-34 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 2 to mira0435601-33 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 1 to mira0435601-35 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 1 to mira0435601-36 2014-07-01 16:16:06,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 2 to mira0435601-35 2014-07-01 16:16:06,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 4 to mira0435601-35 2014-07-01 16:16:06,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: done (15 left) 2014-07-01 16:16:06,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: write initial oid 42 2014-07-01 16:16:06,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: seq_num 41 ranges {0=532480,532480=507904,1040384=614400,1654784=442368,2097152=40960} 2014-07-01 16:16:06,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0435601-42 from 0 to 532480 tid 1 2014-07-01 16:16:06,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0435601-42 from 532480 to 1040384 tid 2 2014-07-01 16:16:06,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0435601-42 from 1040384 to 1654784 tid 3 2014-07-01 16:16:06,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0435601-42 from 1654784 to 2097152 tid 4 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0435601-42 from 2097152 to 2138112 tid 5 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 2 to mira0435601-36 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 3 to mira0435601-33 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 5 to mira0435601-33 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 6 to mira0435601-33 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 13 (ObjNum 32 snap 0 seq_num 32) dirty exists 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 1 to mira0435601-39 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 3 to mira0435601-36 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 3 to mira0435601-39 2014-07-01 16:16:06,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 1 to mira0435601-40 2014-07-01 16:16:06,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 3 to mira0435601-40 2014-07-01 16:16:06,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: done (15 left) 2014-07-01 16:16:06,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: write initial oid 43 2014-07-01 16:16:06,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: seq_num 42 ranges {0=405504,405504=495616,901120=688128,1589248=421888} 2014-07-01 16:16:06,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: writing mira0435601-43 from 0 to 405504 tid 1 2014-07-01 16:16:06,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: writing mira0435601-43 from 405504 to 901120 tid 2 2014-07-01 16:16:06,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: writing mira0435601-43 from 901120 to 1589248 tid 3 2014-07-01 16:16:06,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: writing mira0435601-43 from 1589248 to 2011136 tid 4 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 4 to mira0435601-36 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 6 to mira0435601-36 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 7 to mira0435601-36 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 36 v 9 (ObjNum 35 snap 0 seq_num 35) dirty exists 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 1 to mira0435601-37 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 3 to mira0435601-37 2014-07-01 16:16:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 1 to mira0435601-43 2014-07-01 16:16:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: done (15 left) 2014-07-01 16:16:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: write initial oid 44 2014-07-01 16:16:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: seq_num 43 ranges {0=434176,434176=401408} 2014-07-01 16:16:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: writing mira0435601-44 from 0 to 434176 tid 1 2014-07-01 16:16:06,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: writing mira0435601-44 from 434176 to 835584 tid 2 2014-07-01 16:16:06,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:06,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 1 to mira0435601-42 2014-07-01 16:16:06,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 2 to mira0435601-43 2014-07-01 16:16:06,826.826 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:16:06,840.840 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:16:06,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 1 to mira0435601-44 2014-07-01 16:16:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 3 to mira0435601-43 2014-07-01 16:16:06,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 2 to mira0435601-42 2014-07-01 16:16:06,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 2 to mira0435601-44 2014-07-01 16:16:06,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 4 to mira0435601-44 2014-07-01 16:16:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 4 to mira0435601-43 2014-07-01 16:16:07,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 6 to mira0435601-43 2014-07-01 16:16:07,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 5 to mira0435601-44 2014-07-01 16:16:07,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 44 v 12 (ObjNum 43 snap 0 seq_num 43) dirty exists 2014-07-01 16:16:07,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: done (15 left) 2014-07-01 16:16:07,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: write initial oid 45 2014-07-01 16:16:07,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: seq_num 44 ranges {0=704512} 2014-07-01 16:16:07,175.175 INFO:teuthology.orchestra.run.mira028:Running: 'lsb_release -sc' 2014-07-01 16:16:07,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: writing mira0435601-45 from 0 to 704512 tid 1 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 7 to mira0435601-43 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 11 (ObjNum 42 snap 0 seq_num 42) dirty exists 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: done (15 left) 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: write initial oid 46 2014-07-01 16:16:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: seq_num 45 ranges {0=733184,733184=491520,1224704=614400,1839104=98304} 2014-07-01 16:16:07,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0435601-46 from 0 to 733184 tid 1 2014-07-01 16:16:07,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0435601-46 from 733184 to 1224704 tid 2 2014-07-01 16:16:07,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0435601-46 from 1224704 to 1839104 tid 3 2014-07-01 16:16:07,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0435601-46 from 1839104 to 1937408 tid 4 2014-07-01 16:16:07,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 3 to mira0435601-42 2014-07-01 16:16:07,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 1 to mira0435601-46 2014-07-01 16:16:07,264.264 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1 16 25 9 35.9889 36 0.900233 0.679322 2014-07-01 16:16:07,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 2 to mira0435601-46 2014-07-01 16:16:07,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 3 to mira0435601-46 2014-07-01 16:16:07,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 4 to mira0435601-46 2014-07-01 16:16:07,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 6 to mira0435601-46 2014-07-01 16:16:07,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 7 to mira0435601-46 2014-07-01 16:16:07,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 16 (ObjNum 45 snap 0 seq_num 45) dirty exists 2014-07-01 16:16:07,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: done (15 left) 2014-07-01 16:16:07,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: write initial oid 47 2014-07-01 16:16:07,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: seq_num 46 ranges {0=249856} 2014-07-01 16:16:07,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: writing mira0435601-47 from 0 to 249856 tid 1 2014-07-01 16:16:07,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 4 to mira0435601-42 2014-07-01 16:16:07,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 5 to mira0435601-42 2014-07-01 16:16:07,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 7 to mira0435601-42 2014-07-01 16:16:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 8 to mira0435601-42 2014-07-01 16:16:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 19 (ObjNum 41 snap 0 seq_num 41) dirty exists 2014-07-01 16:16:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: done (15 left) 2014-07-01 16:16:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: write initial oid 48 2014-07-01 16:16:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: seq_num 47 ranges {0=122880} 2014-07-01 16:16:07,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: writing mira0435601-48 from 0 to 122880 tid 1 2014-07-01 16:16:07,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 1 to mira0435601-48 2014-07-01 16:16:07,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 3 to mira0435601-48 2014-07-01 16:16:07,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 4 to mira0435601-48 2014-07-01 16:16:07,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 14 (ObjNum 47 snap 0 seq_num 47) dirty exists 2014-07-01 16:16:07,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: done (15 left) 2014-07-01 16:16:07,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: write initial oid 49 2014-07-01 16:16:07,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: seq_num 48 ranges {0=552960,552960=487424,1040384=315392} 2014-07-01 16:16:07,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0435601-49 from 0 to 552960 tid 1 2014-07-01 16:16:07,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0435601-49 from 552960 to 1040384 tid 2 2014-07-01 16:16:07,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0435601-49 from 1040384 to 1355776 tid 3 2014-07-01 16:16:07,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 1 to mira0435601-45 2014-07-01 16:16:07,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 3 to mira0435601-45 2014-07-01 16:16:07,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 4 to mira0435601-45 2014-07-01 16:16:07,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 21 (ObjNum 44 snap 0 seq_num 44) dirty exists 2014-07-01 16:16:07,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: done (15 left) 2014-07-01 16:16:07,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: write initial oid 50 2014-07-01 16:16:07,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: seq_num 49 ranges {0=421888,421888=716800,1138688=86016} 2014-07-01 16:16:07,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0435601-50 from 0 to 421888 tid 1 2014-07-01 16:16:07,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0435601-50 from 421888 to 1138688 tid 2 2014-07-01 16:16:07,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0435601-50 from 1138688 to 1224704 tid 3 2014-07-01 16:16:07,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:07,702.702 INFO:teuthology.orchestra.run.mira028:Running: 'arch' 2014-07-01 16:16:07,845.845 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:16:07,861.861 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:16:07,862.862 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:07,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 1 to mira0435601-47 2014-07-01 16:16:07,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 3 to mira0435601-47 2014-07-01 16:16:07,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 4 to mira0435601-47 2014-07-01 16:16:07,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 23 (ObjNum 46 snap 0 seq_num 46) dirty exists 2014-07-01 16:16:07,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: done (15 left) 2014-07-01 16:16:07,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: write initial oid 51 2014-07-01 16:16:07,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: seq_num 50 ranges {0=745472,745472=700416,1445888=503808} 2014-07-01 16:16:07,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: writing mira0435601-51 from 0 to 745472 tid 1 2014-07-01 16:16:07,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: writing mira0435601-51 from 745472 to 1445888 tid 2 2014-07-01 16:16:07,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: writing mira0435601-51 from 1445888 to 1949696 tid 3 2014-07-01 16:16:07,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:08,209.209 INFO:teuthology.task.install:dist precise arch x86_64 2014-07-01 16:16:08,209.209 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac 2014-07-01 16:16:08,210.210 INFO:teuthology.orchestra.run.mira028:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac/version' 2014-07-01 16:16:08,261.261 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 2 16 39 23 45.9874 56 0.441946 0.648225 2014-07-01 16:16:08,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 7 to mira0435601-25 2014-07-01 16:16:08,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 5 (ObjNum 24 snap 0 seq_num 24) dirty exists 2014-07-01 16:16:08,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: done (15 left) 2014-07-01 16:16:08,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: write initial oid 52 2014-07-01 16:16:08,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: seq_num 51 ranges {0=774144} 2014-07-01 16:16:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: writing mira0435601-52 from 0 to 774144 tid 1 2014-07-01 16:16:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 4 to mira0435601-31 2014-07-01 16:16:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 11 (ObjNum 30 snap 0 seq_num 30) dirty exists 2014-07-01 16:16:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 6 to mira0435601-26 2014-07-01 16:16:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 15 (ObjNum 25 snap 0 seq_num 25) dirty exists 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 5 to mira0435601-28 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 28 v 3 (ObjNum 27 snap 0 seq_num 27) dirty exists 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 6 to mira0435601-32 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 32 v 10 (ObjNum 31 snap 0 seq_num 31) dirty exists 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 4 to mira0435601-30 2014-07-01 16:16:08,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 5 (ObjNum 29 snap 0 seq_num 29) dirty exists 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: done (15 left) 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: done (14 left) 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: done (13 left) 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: done (12 left) 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: done (11 left) 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: write initial oid 53 2014-07-01 16:16:08,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: seq_num 52 ranges {0=643072} 2014-07-01 16:16:08,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: writing mira0435601-53 from 0 to 643072 tid 1 2014-07-01 16:16:08,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: write initial oid 54 2014-07-01 16:16:08,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: seq_num 53 ranges {0=516096} 2014-07-01 16:16:08,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: writing mira0435601-54 from 0 to 516096 tid 1 2014-07-01 16:16:08,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: write initial oid 55 2014-07-01 16:16:08,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: seq_num 54 ranges {0=544768,544768=679936,1224704=524288} 2014-07-01 16:16:08,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: writing mira0435601-55 from 0 to 544768 tid 1 2014-07-01 16:16:08,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: writing mira0435601-55 from 544768 to 1224704 tid 2 2014-07-01 16:16:08,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: writing mira0435601-55 from 1224704 to 1748992 tid 3 2014-07-01 16:16:08,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: write initial oid 56 2014-07-01 16:16:08,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: seq_num 55 ranges {0=61440} 2014-07-01 16:16:08,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: writing mira0435601-56 from 0 to 61440 tid 1 2014-07-01 16:16:08,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: write initial oid 57 2014-07-01 16:16:08,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: seq_num 56 ranges {0=737280,737280=446464,1183744=729088,1912832=430080} 2014-07-01 16:16:08,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0435601-57 from 0 to 737280 tid 1 2014-07-01 16:16:08,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0435601-57 from 737280 to 1183744 tid 2 2014-07-01 16:16:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0435601-57 from 1183744 to 1912832 tid 3 2014-07-01 16:16:08,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0435601-57 from 1912832 to 2342912 tid 4 2014-07-01 16:16:08,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:08,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 7 to mira0435601-34 2014-07-01 16:16:08,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 20 (ObjNum 33 snap 0 seq_num 33) dirty exists 2014-07-01 16:16:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 5 to mira0435601-35 2014-07-01 16:16:08,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 3 (ObjNum 34 snap 0 seq_num 34) dirty exists 2014-07-01 16:16:08,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 4 to mira0435601-39 2014-07-01 16:16:08,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 39 v 7 (ObjNum 38 snap 0 seq_num 38) dirty exists 2014-07-01 16:16:08,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 4 to mira0435601-37 2014-07-01 16:16:08,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 2 (ObjNum 36 snap 0 seq_num 36) dirty exists 2014-07-01 16:16:08,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 4 to mira0435601-40 2014-07-01 16:16:08,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 9 (ObjNum 39 snap 0 seq_num 39) dirty exists 2014-07-01 16:16:08,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: done (15 left) 2014-07-01 16:16:08,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: done (14 left) 2014-07-01 16:16:08,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: done (13 left) 2014-07-01 16:16:08,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: done (12 left) 2014-07-01 16:16:08,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: done (11 left) 2014-07-01 16:16:08,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: write initial oid 58 2014-07-01 16:16:08,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: seq_num 57 ranges {0=765952,765952=401408} 2014-07-01 16:16:08,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0435601-58 from 0 to 765952 tid 1 2014-07-01 16:16:08,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0435601-58 from 765952 to 1167360 tid 2 2014-07-01 16:16:08,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: write initial oid 59 2014-07-01 16:16:08,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: seq_num 58 ranges {0=790528,790528=663552,1454080=675840,2129920=266240} 2014-07-01 16:16:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0435601-59 from 0 to 790528 tid 1 2014-07-01 16:16:08,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0435601-59 from 790528 to 1454080 tid 2 2014-07-01 16:16:08,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0435601-59 from 1454080 to 2129920 tid 3 2014-07-01 16:16:08,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0435601-59 from 2129920 to 2396160 tid 4 2014-07-01 16:16:08,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: write initial oid 60 2014-07-01 16:16:08,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: seq_num 59 ranges {0=557056,557056=737280,1294336=466944} 2014-07-01 16:16:08,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: writing mira0435601-60 from 0 to 557056 tid 1 2014-07-01 16:16:08,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: writing mira0435601-60 from 557056 to 1294336 tid 2 2014-07-01 16:16:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: writing mira0435601-60 from 1294336 to 1761280 tid 3 2014-07-01 16:16:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: write initial oid 61 2014-07-01 16:16:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: seq_num 60 ranges {0=585728} 2014-07-01 16:16:08,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: writing mira0435601-61 from 0 to 585728 tid 1 2014-07-01 16:16:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: write initial oid 62 2014-07-01 16:16:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: seq_num 61 ranges {0=610304,610304=503808,1114112=446464,1560576=253952} 2014-07-01 16:16:08,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: writing mira0435601-62 from 0 to 610304 tid 1 2014-07-01 16:16:08,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: writing mira0435601-62 from 610304 to 1114112 tid 2 2014-07-01 16:16:08,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: writing mira0435601-62 from 1114112 to 1560576 tid 3 2014-07-01 16:16:08,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: writing mira0435601-62 from 1560576 to 1814528 tid 4 2014-07-01 16:16:08,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:08,766.766 INFO:teuthology.task.install:Package version is 0.81-365-g0aed380-1precise 2014-07-01 16:16:08,766.766 INFO:teuthology.orchestra.run.mira028:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-precise-x86_64-basic/sha1/0aed380a61114710cf5edf49472d1c91e31991ac precise main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-07-01 16:16:08,773.773 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:09,261.261 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 3 16 39 23 30.6586 0 - 0.648225 2014-07-01 16:16:09,318.318 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:09,346.346 INFO:teuthology.orchestra.run.mira028:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.81-365-g0aed380-1precise ceph-dbg=0.81-365-g0aed380-1precise ceph-mds=0.81-365-g0aed380-1precise ceph-mds-dbg=0.81-365-g0aed380-1precise ceph-common=0.81-365-g0aed380-1precise ceph-common-dbg=0.81-365-g0aed380-1precise ceph-fuse=0.81-365-g0aed380-1precise ceph-fuse-dbg=0.81-365-g0aed380-1precise ceph-test=0.81-365-g0aed380-1precise ceph-test-dbg=0.81-365-g0aed380-1precise radosgw=0.81-365-g0aed380-1precise radosgw-dbg=0.81-365-g0aed380-1precise python-ceph=0.81-365-g0aed380-1precise libcephfs1=0.81-365-g0aed380-1precise libcephfs1-dbg=0.81-365-g0aed380-1precise libcephfs-java=0.81-365-g0aed380-1precise librados2=0.81-365-g0aed380-1precise librados2-dbg=0.81-365-g0aed380-1precise librbd1=0.81-365-g0aed380-1precise librbd1-dbg=0.81-365-g0aed380-1precise' 2014-07-01 16:16:09,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 1 to mira0435601-41 2014-07-01 16:16:09,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 2 to mira0435601-41 2014-07-01 16:16:09,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 3 to mira0435601-41 2014-07-01 16:16:09,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 1 to mira0435601-52 2014-07-01 16:16:09,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 5 to mira0435601-41 2014-07-01 16:16:09,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 3 to mira0435601-52 2014-07-01 16:16:09,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 1 to mira0435601-49 2014-07-01 16:16:09,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 1 to mira0435601-38 2014-07-01 16:16:09,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 3 to mira0435601-38 2014-07-01 16:16:09,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 2 to mira0435601-49 2014-07-01 16:16:09,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 3 to mira0435601-49 2014-07-01 16:16:09,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 5 to mira0435601-49 2014-07-01 16:16:09,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 1 to mira0435601-50 2014-07-01 16:16:09,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 2 to mira0435601-50 2014-07-01 16:16:09,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 3 to mira0435601-50 2014-07-01 16:16:09,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 5 to mira0435601-50 2014-07-01 16:16:09,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 1 to mira0435601-54 2014-07-01 16:16:09,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 3 to mira0435601-54 2014-07-01 16:16:09,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 4 to mira0435601-38 2014-07-01 16:16:09,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 13 (ObjNum 37 snap 0 seq_num 37) dirty exists 2014-07-01 16:16:09,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: done (15 left) 2014-07-01 16:16:09,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: write initial oid 63 2014-07-01 16:16:09,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: seq_num 62 ranges {0=327680} 2014-07-01 16:16:09,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: writing mira0435601-63 from 0 to 327680 tid 1 2014-07-01 16:16:09,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 1 to mira0435601-58 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 2 to mira0435601-58 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 6 to mira0435601-41 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 9 (ObjNum 40 snap 0 seq_num 40) dirty exists 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 1 to mira0435601-51 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 4 to mira0435601-58 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: done (15 left) 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: write initial oid 64 2014-07-01 16:16:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: seq_num 63 ranges {0=757760,757760=716800,1474560=86016} 2014-07-01 16:16:09,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0435601-64 from 0 to 757760 tid 1 2014-07-01 16:16:09,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0435601-64 from 757760 to 1474560 tid 2 2014-07-01 16:16:09,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0435601-64 from 1474560 to 1560576 tid 3 2014-07-01 16:16:09,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:09,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 1 to mira0435601-61 2014-07-01 16:16:09,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 3 to mira0435601-61 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 2 to mira0435601-51 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 3 to mira0435601-51 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 1 to mira0435601-57 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 5 to mira0435601-51 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 1 to mira0435601-55 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 1 to mira0435601-53 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 2 to mira0435601-57 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 3 to mira0435601-53 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 2 to mira0435601-55 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 3 to mira0435601-57 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 1 to mira0435601-56 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 3 to mira0435601-55 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 5 to mira0435601-55 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 6 to mira0435601-49 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 49 v 13 (ObjNum 48 snap 0 seq_num 48) dirty exists 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 4 to mira0435601-57 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 3 to mira0435601-56 2014-07-01 16:16:09,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 6 to mira0435601-57 2014-07-01 16:16:09,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 1 to mira0435601-59 2014-07-01 16:16:09,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 2 to mira0435601-59 2014-07-01 16:16:09,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: done (15 left) 2014-07-01 16:16:09,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: write initial oid 65 2014-07-01 16:16:09,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: seq_num 64 ranges {0=430080,430080=704512,1134592=98304} 2014-07-01 16:16:09,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: writing mira0435601-65 from 0 to 430080 tid 1 2014-07-01 16:16:09,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: writing mira0435601-65 from 430080 to 1134592 tid 2 2014-07-01 16:16:09,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: writing mira0435601-65 from 1134592 to 1232896 tid 3 2014-07-01 16:16:09,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 3 to mira0435601-59 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 4 to mira0435601-59 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 1 to mira0435601-60 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 2 to mira0435601-60 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 3 to mira0435601-60 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 6 to mira0435601-51 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 51 v 4 (ObjNum 50 snap 0 seq_num 50) dirty exists 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 5 to mira0435601-60 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 6 to mira0435601-59 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: done (15 left) 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: write initial oid 66 2014-07-01 16:16:09,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: seq_num 65 ranges {0=548864,548864=483328,1032192=446464,1478656=483328,1961984=192512} 2014-07-01 16:16:09,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0435601-66 from 0 to 548864 tid 1 2014-07-01 16:16:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0435601-66 from 548864 to 1032192 tid 2 2014-07-01 16:16:09,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0435601-66 from 1032192 to 1478656 tid 3 2014-07-01 16:16:09,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0435601-66 from 1478656 to 1961984 tid 4 2014-07-01 16:16:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0435601-66 from 1961984 to 2154496 tid 5 2014-07-01 16:16:09,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:09,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 1 to mira0435601-62 2014-07-01 16:16:09,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 2 to mira0435601-62 2014-07-01 16:16:09,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 3 to mira0435601-62 2014-07-01 16:16:09,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 4 to mira0435601-62 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 6 to mira0435601-62 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 1 to mira0435601-63 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 7 to mira0435601-62 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 23 (ObjNum 61 snap 0 seq_num 61) dirty exists 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 3 to mira0435601-63 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: done (15 left) 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: write initial oid 67 2014-07-01 16:16:09,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: seq_num 66 ranges {0=573440,573440=401408} 2014-07-01 16:16:09,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: writing mira0435601-67 from 0 to 573440 tid 1 2014-07-01 16:16:09,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: writing mira0435601-67 from 573440 to 974848 tid 2 2014-07-01 16:16:09,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:10,058.058 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:10,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 1 to mira0435601-64 2014-07-01 16:16:10,264.264 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 4 16 57 41 40.99 36 2.67815 1.16897 2014-07-01 16:16:10,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 4 to mira0435601-54 2014-07-01 16:16:10,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 12 (ObjNum 53 snap 0 seq_num 53) dirty exists 2014-07-01 16:16:10,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: done (15 left) 2014-07-01 16:16:10,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: write initial oid 68 2014-07-01 16:16:10,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: seq_num 67 ranges {0=602112,602112=700416,1302528=794624,2097152=110592} 2014-07-01 16:16:10,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0435601-68 from 0 to 602112 tid 1 2014-07-01 16:16:10,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0435601-68 from 602112 to 1302528 tid 2 2014-07-01 16:16:10,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0435601-68 from 1302528 to 2097152 tid 3 2014-07-01 16:16:10,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0435601-68 from 2097152 to 2207744 tid 4 2014-07-01 16:16:10,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:10,794.794 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:16:10,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 2 to mira0435601-64 2014-07-01 16:16:10,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 3 to mira0435601-64 2014-07-01 16:16:10,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 5 to mira0435601-64 2014-07-01 16:16:10,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 1 to mira0435601-65 2014-07-01 16:16:10,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 2 to mira0435601-65 2014-07-01 16:16:10,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 3 to mira0435601-65 2014-07-01 16:16:10,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 5 to mira0435601-65 2014-07-01 16:16:10,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 1 to mira0435601-66 2014-07-01 16:16:10,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 2 to mira0435601-66 2014-07-01 16:16:10,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 4 to mira0435601-52 2014-07-01 16:16:10,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 52 v 16 (ObjNum 51 snap 0 seq_num 51) dirty exists 2014-07-01 16:16:10,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: done (15 left) 2014-07-01 16:16:10,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: write initial oid 69 2014-07-01 16:16:10,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: seq_num 68 ranges {0=770048,770048=684032,1454080=118784} 2014-07-01 16:16:10,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: writing mira0435601-69 from 0 to 770048 tid 1 2014-07-01 16:16:10,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: writing mira0435601-69 from 770048 to 1454080 tid 2 2014-07-01 16:16:11,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: writing mira0435601-69 from 1454080 to 1572864 tid 3 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 6 to mira0435601-50 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 24 (ObjNum 49 snap 0 seq_num 49) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 5 to mira0435601-58 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 5 (ObjNum 57 snap 0 seq_num 57) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 4 to mira0435601-56 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 9 (ObjNum 55 snap 0 seq_num 55) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 4 to mira0435601-61 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 18 (ObjNum 60 snap 0 seq_num 60) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 4 to mira0435601-53 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 6 (ObjNum 52 snap 0 seq_num 52) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 7 to mira0435601-57 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 21 (ObjNum 56 snap 0 seq_num 56) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 7 to mira0435601-59 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 14 (ObjNum 58 snap 0 seq_num 58) dirty exists 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: done (15 left) 2014-07-01 16:16:11,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: done (14 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: done (13 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: done (12 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: done (11 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: done (10 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: done (9 left) 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: write initial oid 70 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: seq_num 69 ranges {0=393216} 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: writing mira0435601-70 from 0 to 393216 tid 1 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: write initial oid 71 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: seq_num 70 ranges {0=421888,421888=696320,1118208=413696,1531904=94208} 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0435601-71 from 0 to 421888 tid 1 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0435601-71 from 421888 to 1118208 tid 2 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0435601-71 from 1118208 to 1531904 tid 3 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0435601-71 from 1531904 to 1626112 tid 4 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: write initial oid 72 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: seq_num 71 ranges {0=696320,696320=524288,1220608=278528} 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: writing mira0435601-72 from 0 to 696320 tid 1 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: writing mira0435601-72 from 696320 to 1220608 tid 2 2014-07-01 16:16:11,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: writing mira0435601-72 from 1220608 to 1499136 tid 3 2014-07-01 16:16:11,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: write initial oid 73 2014-07-01 16:16:11,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: seq_num 72 ranges {0=565248,565248=507904,1073152=294912} 2014-07-01 16:16:11,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0435601-73 from 0 to 565248 tid 1 2014-07-01 16:16:11,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0435601-73 from 565248 to 1073152 tid 2 2014-07-01 16:16:11,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0435601-73 from 1073152 to 1368064 tid 3 2014-07-01 16:16:11,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: write initial oid 74 2014-07-01 16:16:11,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: seq_num 73 ranges {0=643072,643072=401408} 2014-07-01 16:16:11,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: writing mira0435601-74 from 0 to 643072 tid 1 2014-07-01 16:16:11,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: writing mira0435601-74 from 643072 to 1044480 tid 2 2014-07-01 16:16:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: write initial oid 75 2014-07-01 16:16:11,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: seq_num 74 ranges {0=516096,516096=401408} 2014-07-01 16:16:11,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: writing mira0435601-75 from 0 to 516096 tid 1 2014-07-01 16:16:11,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: writing mira0435601-75 from 516096 to 917504 tid 2 2014-07-01 16:16:11,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: write initial oid 76 2014-07-01 16:16:11,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: seq_num 75 ranges {0=786432} 2014-07-01 16:16:11,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: writing mira0435601-76 from 0 to 786432 tid 1 2014-07-01 16:16:11,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:11,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 6 to mira0435601-60 2014-07-01 16:16:11,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 60 v 25 (ObjNum 59 snap 0 seq_num 59) dirty exists 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 4 to mira0435601-63 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 27 (ObjNum 62 snap 0 seq_num 62) dirty exists 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 6 to mira0435601-64 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 10 (ObjNum 63 snap 0 seq_num 63) dirty exists 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 6 to mira0435601-65 2014-07-01 16:16:11,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 13 (ObjNum 64 snap 0 seq_num 64) dirty exists 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 6 to mira0435601-55 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 6 (ObjNum 54 snap 0 seq_num 54) dirty exists 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: done (15 left) 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: done (14 left) 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: done (13 left) 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: done (12 left) 2014-07-01 16:16:11,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: done (11 left) 2014-07-01 16:16:11,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: write initial oid 77 2014-07-01 16:16:11,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: seq_num 76 ranges {0=413696,413696=491520,905216=409600,1314816=622592,1937408=81920} 2014-07-01 16:16:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0435601-77 from 0 to 413696 tid 1 2014-07-01 16:16:11,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0435601-77 from 413696 to 905216 tid 2 2014-07-01 16:16:11,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0435601-77 from 905216 to 1314816 tid 3 2014-07-01 16:16:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0435601-77 from 1314816 to 1937408 tid 4 2014-07-01 16:16:11,262.262 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 5 16 60 44 35.1919 12 1.1119 1.21659 2014-07-01 16:16:11,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0435601-77 from 1937408 to 2019328 tid 5 2014-07-01 16:16:11,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: write initial oid 78 2014-07-01 16:16:11,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: seq_num 77 ranges {0=335872} 2014-07-01 16:16:11,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: writing mira0435601-78 from 0 to 335872 tid 1 2014-07-01 16:16:11,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: write initial oid 79 2014-07-01 16:16:11,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: seq_num 78 ranges {0=204800} 2014-07-01 16:16:11,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: writing mira0435601-79 from 0 to 204800 tid 1 2014-07-01 16:16:11,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: write initial oid 80 2014-07-01 16:16:11,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: seq_num 79 ranges {0=634880,634880=483328,1118208=319488} 2014-07-01 16:16:11,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: writing mira0435601-80 from 0 to 634880 tid 1 2014-07-01 16:16:11,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: writing mira0435601-80 from 634880 to 1118208 tid 2 2014-07-01 16:16:11,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: writing mira0435601-80 from 1118208 to 1437696 tid 3 2014-07-01 16:16:11,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: write initial oid 81 2014-07-01 16:16:11,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: seq_num 80 ranges {0=507904,507904=471040,978944=331776} 2014-07-01 16:16:11,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0435601-81 from 0 to 507904 tid 1 2014-07-01 16:16:11,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0435601-81 from 507904 to 978944 tid 2 2014-07-01 16:16:11,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0435601-81 from 978944 to 1310720 tid 3 2014-07-01 16:16:11,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:11,532.532 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 8, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 8, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003191', u'skew': u'-0.013913', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002078', u'skew': u'-0.014466', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'25 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'25 pgs stuck unclean'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'num_pgs': 104, u'data_bytes': 54298460, u'bytes_used': 791257088, u'op_per_sec': 6, u'version': 38, u'pgs_by_state': [{u'count': 25, u'state_name': u'creating'}, {u'count': 1, u'state_name': u'active+clean+replay'}, {u'count': 78, u'state_name': u'active+clean'}], u'read_bytes_sec': 987, u'write_bytes_sec': 13095449, u'bytes_avail': 4997791281152}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 23, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:16:11,532.532 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:16:12,262.262 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 6 16 60 44 29.3266 0 - 1.21659 2014-07-01 16:16:12,282.282 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:16:12,282.282 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:13,044.044 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:13,261.261 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 7 16 60 44 25.1372 0 - 1.21659 2014-07-01 16:16:14,263.263 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 8 16 60 44 21.995 0 - 1.21659 2014-07-01 16:16:15,264.264 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 9 16 60 44 19.5511 0 - 1.21659 2014-07-01 16:16:15,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 3 to mira0435601-66 2014-07-01 16:16:15,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 4 to mira0435601-66 2014-07-01 16:16:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 1 to mira0435601-76 2014-07-01 16:16:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 3 to mira0435601-76 2014-07-01 16:16:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 4 to mira0435601-76 2014-07-01 16:16:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 2 (ObjNum 75 snap 0 seq_num 75) dirty exists 2014-07-01 16:16:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: done (15 left) 2014-07-01 16:16:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: write initial oid 82 2014-07-01 16:16:15,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: seq_num 81 ranges {0=778240,778240=401408} 2014-07-01 16:16:15,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0435601-82 from 0 to 778240 tid 1 2014-07-01 16:16:15,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0435601-82 from 778240 to 1179648 tid 2 2014-07-01 16:16:15,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:16,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 10 16 76 60 23.9946 12.8 5.85324 2.34643 2014-07-01 16:16:16,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 1 to mira0435601-74 2014-07-01 16:16:16,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 2 to mira0435601-74 2014-07-01 16:16:16,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 4 to mira0435601-74 2014-07-01 16:16:16,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 1 to mira0435601-75 2014-07-01 16:16:16,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 2 to mira0435601-75 2014-07-01 16:16:16,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 4 to mira0435601-75 2014-07-01 16:16:16,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 1 to mira0435601-81 2014-07-01 16:16:16,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 2 to mira0435601-81 2014-07-01 16:16:16,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 3 to mira0435601-81 2014-07-01 16:16:16,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 5 to mira0435601-81 2014-07-01 16:16:16,758.758 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:16,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 5 to mira0435601-66 2014-07-01 16:16:16,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 1 to mira0435601-67 2014-07-01 16:16:16,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 2 to mira0435601-67 2014-07-01 16:16:16,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 1 to mira0435601-71 2014-07-01 16:16:16,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 2 to mira0435601-71 2014-07-01 16:16:16,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 4 to mira0435601-67 2014-07-01 16:16:16,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 7 to mira0435601-66 2014-07-01 16:16:16,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 1 to mira0435601-78 2014-07-01 16:16:16,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 3 to mira0435601-71 2014-07-01 16:16:16,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 4 to mira0435601-71 2014-07-01 16:16:16,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 1 to mira0435601-69 2014-07-01 16:16:16,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 6 to mira0435601-71 2014-07-01 16:16:16,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 2 to mira0435601-69 2014-07-01 16:16:16,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 1 to mira0435601-77 2014-07-01 16:16:16,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 3 to mira0435601-69 2014-07-01 16:16:16,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 2 to mira0435601-77 2014-07-01 16:16:16,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 1 to mira0435601-72 2014-07-01 16:16:16,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 3 to mira0435601-77 2014-07-01 16:16:16,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 5 to mira0435601-69 2014-07-01 16:16:16,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 2 to mira0435601-72 2014-07-01 16:16:16,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 4 to mira0435601-77 2014-07-01 16:16:16,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 3 to mira0435601-72 2014-07-01 16:16:16,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 5 to mira0435601-77 2014-07-01 16:16:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 5 to mira0435601-72 2014-07-01 16:16:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 1 to mira0435601-73 2014-07-01 16:16:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 7 to mira0435601-77 2014-07-01 16:16:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 2 to mira0435601-73 2014-07-01 16:16:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 3 to mira0435601-73 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 5 to mira0435601-73 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 1 to mira0435601-79 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 3 to mira0435601-78 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 5 to mira0435601-74 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 16 (ObjNum 73 snap 0 seq_num 73) dirty exists 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: done (15 left) 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: write initial oid 83 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: seq_num 82 ranges {0=700416,700416=479232,1179648=507904,1687552=217088} 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: writing mira0435601-83 from 0 to 700416 tid 1 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: writing mira0435601-83 from 700416 to 1179648 tid 2 2014-07-01 16:16:16,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: writing mira0435601-83 from 1179648 to 1687552 tid 3 2014-07-01 16:16:16,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: writing mira0435601-83 from 1687552 to 1904640 tid 4 2014-07-01 16:16:16,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:16,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 5 to mira0435601-75 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 19 (ObjNum 74 snap 0 seq_num 74) dirty exists 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 6 to mira0435601-81 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 81 v 7 (ObjNum 80 snap 0 seq_num 80) dirty exists 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: done (15 left) 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: done (14 left) 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: write initial oid 84 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: seq_num 83 ranges {0=729088} 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: writing mira0435601-84 from 0 to 729088 tid 1 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: write initial oid 85 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: seq_num 84 ranges {0=753664,753664=696320,1449984=454656,1904640=53248} 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: writing mira0435601-85 from 0 to 753664 tid 1 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: writing mira0435601-85 from 753664 to 1449984 tid 2 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: writing mira0435601-85 from 1449984 to 1904640 tid 3 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: writing mira0435601-85 from 1904640 to 1957888 tid 4 2014-07-01 16:16:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:17,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 1 to mira0435601-80 2014-07-01 16:16:17,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 2 to mira0435601-80 2014-07-01 16:16:17,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 1 to mira0435601-68 2014-07-01 16:16:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 3 to mira0435601-79 2014-07-01 16:16:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 2 to mira0435601-68 2014-07-01 16:16:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 3 to mira0435601-80 2014-07-01 16:16:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 3 to mira0435601-68 2014-07-01 16:16:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 4 to mira0435601-68 2014-07-01 16:16:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 6 to mira0435601-68 2014-07-01 16:16:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 1 to mira0435601-70 2014-07-01 16:16:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 3 to mira0435601-70 2014-07-01 16:16:17,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 8 to mira0435601-66 2014-07-01 16:16:17,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 16 (ObjNum 65 snap 0 seq_num 65) dirty exists 2014-07-01 16:16:17,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: done (15 left) 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: write initial oid 86 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: seq_num 85 ranges {0=471040} 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: writing mira0435601-86 from 0 to 471040 tid 1 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 5 to mira0435601-67 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 27 (ObjNum 66 snap 0 seq_num 66) dirty exists 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 6 to mira0435601-69 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 69 v 20 (ObjNum 68 snap 0 seq_num 68) dirty exists 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 7 to mira0435601-68 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 10 (ObjNum 67 snap 0 seq_num 67) dirty exists 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 4 to mira0435601-70 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 70 v 15 (ObjNum 69 snap 0 seq_num 69) dirty exists 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: done (15 left) 2014-07-01 16:16:17,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: done (14 left) 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: done (13 left) 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: done (12 left) 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: write initial oid 87 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: seq_num 86 ranges {0=143360} 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: writing mira0435601-87 from 0 to 143360 tid 1 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: write initial oid 88 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: seq_num 87 ranges {0=573440,573440=692224,1265664=110592} 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: writing mira0435601-88 from 0 to 573440 tid 1 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: writing mira0435601-88 from 573440 to 1265664 tid 2 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: writing mira0435601-88 from 1265664 to 1376256 tid 3 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: write initial oid 89 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: seq_num 88 ranges {0=692224,692224=675840,1368064=503808,1871872=425984} 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0435601-89 from 0 to 692224 tid 1 2014-07-01 16:16:17,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0435601-89 from 692224 to 1368064 tid 2 2014-07-01 16:16:17,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 11 16 89 73 26.5397 52 1.74042 2.28739 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0435601-89 from 1368064 to 1871872 tid 3 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0435601-89 from 1871872 to 2297856 tid 4 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: write initial oid 90 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: seq_num 89 ranges {0=716800,716800=401408} 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: writing mira0435601-90 from 0 to 716800 tid 1 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: writing mira0435601-90 from 716800 to 1118208 tid 2 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 6 to mira0435601-72 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 18 (ObjNum 71 snap 0 seq_num 71) dirty exists 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 6 to mira0435601-73 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 24 (ObjNum 72 snap 0 seq_num 72) dirty exists 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 4 to mira0435601-78 2014-07-01 16:16:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 29 (ObjNum 77 snap 0 seq_num 77) dirty exists 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: done (15 left) 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: done (14 left) 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: done (13 left) 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: write initial oid 91 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: seq_num 90 ranges {0=745472,745472=442368,1187840=802816,1990656=360448} 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0435601-91 from 0 to 745472 tid 1 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0435601-91 from 745472 to 1187840 tid 2 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0435601-91 from 1187840 to 1990656 tid 3 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0435601-91 from 1990656 to 2351104 tid 4 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: write initial oid 92 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: seq_num 91 ranges {0=512000,512000=516096,1028096=679936,1708032=8192} 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0435601-92 from 0 to 512000 tid 1 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0435601-92 from 512000 to 1028096 tid 2 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0435601-92 from 1028096 to 1708032 tid 3 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0435601-92 from 1708032 to 1716224 tid 4 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: write initial oid 93 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: seq_num 92 ranges {0=536576} 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: writing mira0435601-93 from 0 to 536576 tid 1 2014-07-01 16:16:17,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 7 to mira0435601-71 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 12 (ObjNum 70 snap 0 seq_num 70) dirty exists 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 8 to mira0435601-77 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 18 (ObjNum 76 snap 0 seq_num 76) dirty exists 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 4 to mira0435601-79 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 20 (ObjNum 78 snap 0 seq_num 78) dirty exists 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: done (15 left) 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: done (14 left) 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: done (13 left) 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: write initial oid 94 2014-07-01 16:16:17,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: seq_num 93 ranges {0=565248,565248=684032,1249280=520192} 2014-07-01 16:16:17,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: writing mira0435601-94 from 0 to 565248 tid 1 2014-07-01 16:16:17,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: writing mira0435601-94 from 565248 to 1249280 tid 2 2014-07-01 16:16:17,562.562 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:17,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: writing mira0435601-94 from 1249280 to 1769472 tid 3 2014-07-01 16:16:17,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: write initial oid 95 2014-07-01 16:16:17,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: seq_num 94 ranges {0=438272,438272=667648,1105920=450560,1556480=86016} 2014-07-01 16:16:17,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0435601-95 from 0 to 438272 tid 1 2014-07-01 16:16:17,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0435601-95 from 438272 to 1105920 tid 2 2014-07-01 16:16:17,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0435601-95 from 1105920 to 1556480 tid 3 2014-07-01 16:16:17,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0435601-95 from 1556480 to 1642496 tid 4 2014-07-01 16:16:17,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: write initial oid 96 2014-07-01 16:16:17,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: seq_num 95 ranges {0=757760,757760=499712,1257472=520192,1777664=585728} 2014-07-01 16:16:17,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0435601-96 from 0 to 757760 tid 1 2014-07-01 16:16:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0435601-96 from 757760 to 1257472 tid 2 2014-07-01 16:16:17,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0435601-96 from 1257472 to 1777664 tid 3 2014-07-01 16:16:17,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0435601-96 from 1777664 to 2363392 tid 4 2014-07-01 16:16:17,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:18,115.115 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:16:18,263.263 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 12 16 89 73 24.328 0 - 2.28739 2014-07-01 16:16:18,804.804 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 8, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 8, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003191', u'skew': u'-0.013913', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002078', u'skew': u'-0.014466', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 4998582538240, u'num_pgs': 104, u'data_bytes': 345687900, u'bytes_used': 1690238976, u'op_per_sec': 5, u'version': 41, u'pgs_by_state': [{u'count': 104, u'state_name': u'active+clean'}], u'read_bytes_sec': 409, u'write_bytes_sec': 12486049, u'bytes_avail': 4996892299264}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 23, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:16:18,804.804 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:16:18,804.804 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:18,804.804 INFO:teuthology.task.thrashosds.thrasher:Adding osd 2 2014-07-01 16:16:18,804.804 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 2' 2014-07-01 16:16:19,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 5 to mira0435601-80 2014-07-01 16:16:19,263.263 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 13 16 89 73 22.4566 0 - 2.28739 2014-07-01 16:16:19,651.651 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.2. 2014-07-01 16:16:19,662.662 INFO:teuthology.task.thrashosds.thrasher:Added osd 2 2014-07-01 16:16:20,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 14 16 89 73 20.8526 0 - 2.28739 2014-07-01 16:16:21,264.264 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 15 16 89 73 19.4624 0 - 2.28739 2014-07-01 16:16:22,268.268 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 16 16 89 73 18.246 0 - 2.28739 2014-07-01 16:16:23,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 17 16 89 73 17.1727 0 - 2.28739 2014-07-01 16:16:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 1 to mira0435601-84 2014-07-01 16:16:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 3 to mira0435601-84 2014-07-01 16:16:23,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 4 to mira0435601-84 2014-07-01 16:16:23,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 25 (ObjNum 83 snap 0 seq_num 83) dirty exists 2014-07-01 16:16:23,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: done (15 left) 2014-07-01 16:16:23,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: write initial oid 97 2014-07-01 16:16:23,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: seq_num 96 ranges {0=786432,786432=401408} 2014-07-01 16:16:23,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: writing mira0435601-97 from 0 to 786432 tid 1 2014-07-01 16:16:23,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: writing mira0435601-97 from 786432 to 1187840 tid 2 2014-07-01 16:16:23,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:24,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 1 to mira0435601-86 2014-07-01 16:16:24,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 3 to mira0435601-86 2014-07-01 16:16:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 1 to mira0435601-89 2014-07-01 16:16:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 2 to mira0435601-89 2014-07-01 16:16:24,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 3 to mira0435601-89 2014-07-01 16:16:24,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 4 to mira0435601-89 2014-07-01 16:16:24,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 6 to mira0435601-89 2014-07-01 16:16:24,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 1 to mira0435601-91 2014-07-01 16:16:24,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 2 to mira0435601-91 2014-07-01 16:16:24,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 3 to mira0435601-91 2014-07-01 16:16:24,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 4 to mira0435601-91 2014-07-01 16:16:24,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 6 to mira0435601-91 2014-07-01 16:16:24,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 1 to mira0435601-97 2014-07-01 16:16:24,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 2 to mira0435601-97 2014-07-01 16:16:24,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 4 to mira0435601-97 2014-07-01 16:16:24,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 1 to mira0435601-82 2014-07-01 16:16:24,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 1 to mira0435601-87 2014-07-01 16:16:24,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 3 to mira0435601-87 2014-07-01 16:16:24,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 1 to mira0435601-83 2014-07-01 16:16:24,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 2 to mira0435601-83 2014-07-01 16:16:24,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 3 to mira0435601-83 2014-07-01 16:16:24,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 4 to mira0435601-83 2014-07-01 16:16:24,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 1 to mira0435601-90 2014-07-01 16:16:24,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 2 to mira0435601-90 2014-07-01 16:16:24,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 6 to mira0435601-83 2014-07-01 16:16:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 1 to mira0435601-88 2014-07-01 16:16:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 4 to mira0435601-90 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 2 to mira0435601-88 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 3 to mira0435601-88 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 5 to mira0435601-88 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 1 to mira0435601-92 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 2 to mira0435601-92 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 3 to mira0435601-92 2014-07-01 16:16:24,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 4 to mira0435601-92 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 1 to mira0435601-93 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 6 to mira0435601-92 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 3 to mira0435601-93 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 1 to mira0435601-94 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 4 to mira0435601-87 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 87 v 31 (ObjNum 86 snap 0 seq_num 86) dirty exists 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: done (15 left) 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: write initial oid 98 2014-07-01 16:16:24,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: seq_num 97 ranges {0=659456,659456=401408} 2014-07-01 16:16:24,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0435601-98 from 0 to 659456 tid 1 2014-07-01 16:16:24,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0435601-98 from 659456 to 1060864 tid 2 2014-07-01 16:16:24,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:24,271.271 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 18 16 107 91 20.2178 10.2857 0.325979 3.04844 2014-07-01 16:16:24,662.662 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:24,662.662 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:24,662.662 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2014-07-01 16:16:24,662.662 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 1' 2014-07-01 16:16:24,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 2 to mira0435601-94 2014-07-01 16:16:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 3 to mira0435601-94 2014-07-01 16:16:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 5 to mira0435601-94 2014-07-01 16:16:25,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 7 to mira0435601-83 2014-07-01 16:16:25,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 29 (ObjNum 82 snap 0 seq_num 82) dirty exists 2014-07-01 16:16:25,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: done (15 left) 2014-07-01 16:16:25,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: write initial oid 99 2014-07-01 16:16:25,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: seq_num 98 ranges {0=528384,528384=401408} 2014-07-01 16:16:25,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: writing mira0435601-99 from 0 to 528384 tid 1 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: writing mira0435601-99 from 528384 to 929792 tid 2 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 6 to mira0435601-88 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 33 (ObjNum 87 snap 0 seq_num 87) dirty exists 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 5 to mira0435601-90 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 90 v 13 (ObjNum 89 snap 0 seq_num 89) dirty exists 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 7 to mira0435601-91 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 91 v 12 (ObjNum 90 snap 0 seq_num 90) dirty exists 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 7 to mira0435601-92 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 23 (ObjNum 91 snap 0 seq_num 91) dirty exists 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 7 to mira0435601-89 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 89 v 24 (ObjNum 88 snap 0 seq_num 88) dirty exists 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 4 to mira0435601-93 2014-07-01 16:16:25,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 15 (ObjNum 92 snap 0 seq_num 92) dirty exists 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: done (15 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: done (14 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: done (13 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: done (12 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: done (11 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: done (10 left) 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: write initial oid 100 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: seq_num 99 ranges {0=557056,557056=479232,1036288=413696,1449984=450560,1900544=262144} 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0435601-100 from 0 to 557056 tid 1 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0435601-100 from 557056 to 1036288 tid 2 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0435601-100 from 1036288 to 1449984 tid 3 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0435601-100 from 1449984 to 1900544 tid 4 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0435601-100 from 1900544 to 2162688 tid 5 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: write initial oid 101 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: seq_num 100 ranges {0=479232} 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: writing mira0435601-101 from 0 to 479232 tid 1 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: write initial oid 102 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: seq_num 101 ranges {0=348160} 2014-07-01 16:16:25,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: writing mira0435601-102 from 0 to 348160 tid 1 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: write initial oid 103 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: seq_num 102 ranges {0=778240,778240=475136,1253376=327680} 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: writing mira0435601-103 from 0 to 778240 tid 1 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: writing mira0435601-103 from 778240 to 1253376 tid 2 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: writing mira0435601-103 from 1253376 to 1581056 tid 3 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: write initial oid 104 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: seq_num 103 ranges {0=651264,651264=704512,1355776=98304} 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: writing mira0435601-104 from 0 to 651264 tid 1 2014-07-01 16:16:25,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: writing mira0435601-104 from 651264 to 1355776 tid 2 2014-07-01 16:16:25,299.299 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 19 16 114 98 20.6271 28 0.129451 2.9371 2014-07-01 16:16:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: writing mira0435601-104 from 1355776 to 1454080 tid 3 2014-07-01 16:16:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: write initial oid 105 2014-07-01 16:16:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: seq_num 104 ranges {0=569344,569344=688128,1257472=536576,1794048=380928} 2014-07-01 16:16:25,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0435601-105 from 0 to 569344 tid 1 2014-07-01 16:16:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0435601-105 from 569344 to 1257472 tid 2 2014-07-01 16:16:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0435601-105 from 1257472 to 1794048 tid 3 2014-07-01 16:16:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0435601-105 from 1794048 to 2174976 tid 4 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 5 to mira0435601-97 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 15 (ObjNum 96 snap 0 seq_num 96) dirty exists 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 6 to mira0435601-94 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 37 (ObjNum 93 snap 0 seq_num 93) dirty exists 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: done (15 left) 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: done (14 left) 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: write initial oid 106 2014-07-01 16:16:25,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: seq_num 105 ranges {0=598016,598016=401408} 2014-07-01 16:16:25,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: writing mira0435601-106 from 0 to 598016 tid 1 2014-07-01 16:16:25,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: writing mira0435601-106 from 598016 to 999424 tid 2 2014-07-01 16:16:25,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: write initial oid 107 2014-07-01 16:16:25,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: seq_num 106 ranges {0=471040,471040=401408} 2014-07-01 16:16:25,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0435601-107 from 0 to 471040 tid 1 2014-07-01 16:16:25,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0435601-107 from 471040 to 872448 tid 2 2014-07-01 16:16:25,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:25,973.973 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 1 (8655362) 2014-07-01 16:16:26,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:16:26.178403min lat: 0.099942 max lat: 9.21429 avg lat: 2.88132 2014-07-01 16:16:26,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:16:26,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 20 16 117 101 20.1956 12 0.467713 2.88132 2014-07-01 16:16:27,266.266 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 21 16 123 107 20.3765 24 0.108372 2.79885 2014-07-01 16:16:27,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 1 to mira0435601-100 2014-07-01 16:16:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 2 to mira0435601-100 2014-07-01 16:16:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 3 to mira0435601-100 2014-07-01 16:16:27,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 4 to mira0435601-100 2014-07-01 16:16:27,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 5 to mira0435601-100 2014-07-01 16:16:27,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 1 to mira0435601-98 2014-07-01 16:16:27,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 7 to mira0435601-100 2014-07-01 16:16:27,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 1 to mira0435601-102 2014-07-01 16:16:27,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 2 to mira0435601-98 2014-07-01 16:16:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 4 to mira0435601-98 2014-07-01 16:16:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 3 to mira0435601-102 2014-07-01 16:16:27,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 8 to mira0435601-100 2014-07-01 16:16:27,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 31 (ObjNum 99 snap 0 seq_num 99) dirty exists 2014-07-01 16:16:27,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: done (15 left) 2014-07-01 16:16:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: write initial oid 108 2014-07-01 16:16:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: seq_num 107 ranges {0=495616,495616=684032,1179648=708608,1888256=212992} 2014-07-01 16:16:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0435601-108 from 0 to 495616 tid 1 2014-07-01 16:16:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0435601-108 from 495616 to 1179648 tid 2 2014-07-01 16:16:27,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0435601-108 from 1179648 to 1888256 tid 3 2014-07-01 16:16:27,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0435601-108 from 1888256 to 2101248 tid 4 2014-07-01 16:16:27,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 4 to mira0435601-102 2014-07-01 16:16:27,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 4 (ObjNum 101 snap 0 seq_num 101) dirty exists 2014-07-01 16:16:27,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 2 to mira0435601-82 2014-07-01 16:16:27,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 4 to mira0435601-82 2014-07-01 16:16:27,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 4 to mira0435601-86 2014-07-01 16:16:27,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 25 (ObjNum 85 snap 0 seq_num 85) dirty exists 2014-07-01 16:16:27,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: done (15 left) 2014-07-01 16:16:27,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: done (14 left) 2014-07-01 16:16:27,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: write initial oid 109 2014-07-01 16:16:27,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: seq_num 108 ranges {0=770048,770048=671744,1441792=532480} 2014-07-01 16:16:27,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: writing mira0435601-109 from 0 to 770048 tid 1 2014-07-01 16:16:27,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: writing mira0435601-109 from 770048 to 1441792 tid 2 2014-07-01 16:16:27,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: writing mira0435601-109 from 1441792 to 1974272 tid 3 2014-07-01 16:16:27,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: write initial oid 110 2014-07-01 16:16:27,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: seq_num 109 ranges {0=286720} 2014-07-01 16:16:27,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: writing mira0435601-110 from 0 to 286720 tid 1 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 1 to mira0435601-101 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 3 to mira0435601-101 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 1 to mira0435601-103 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 1 to mira0435601-99 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 2 to mira0435601-103 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 2 to mira0435601-99 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 3 to mira0435601-103 2014-07-01 16:16:27,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 5 to mira0435601-103 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 1 to mira0435601-104 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 2 to mira0435601-104 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 4 to mira0435601-99 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 3 to mira0435601-104 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 1 to mira0435601-106 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 5 to mira0435601-104 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 1 to mira0435601-107 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 2 to mira0435601-107 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 4 to mira0435601-107 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 2 to mira0435601-106 2014-07-01 16:16:27,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 4 to mira0435601-106 2014-07-01 16:16:27,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 1 to mira0435601-109 2014-07-01 16:16:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 6 to mira0435601-80 2014-07-01 16:16:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 22 (ObjNum 79 snap 0 seq_num 79) dirty exists 2014-07-01 16:16:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: done (15 left) 2014-07-01 16:16:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: write initial oid 111 2014-07-01 16:16:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: seq_num 110 ranges {0=716800,716800=434176,1150976=368640} 2014-07-01 16:16:27,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: writing mira0435601-111 from 0 to 716800 tid 1 2014-07-01 16:16:27,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: writing mira0435601-111 from 716800 to 1150976 tid 2 2014-07-01 16:16:27,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: writing mira0435601-111 from 1150976 to 1519616 tid 3 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 5 to mira0435601-82 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 25 (ObjNum 81 snap 0 seq_num 81) dirty exists 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 1 to mira0435601-85 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 4 to mira0435601-101 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 29 (ObjNum 100 snap 0 seq_num 100) dirty exists 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 2 to mira0435601-85 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 3 to mira0435601-85 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 4 to mira0435601-85 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 6 to mira0435601-85 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 5 to mira0435601-98 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 27 (ObjNum 97 snap 0 seq_num 97) dirty exists 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 1 to mira0435601-108 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: done (15 left) 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: done (14 left) 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: done (13 left) 2014-07-01 16:16:27,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: write initial oid 112 2014-07-01 16:16:27,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: seq_num 111 ranges {0=589824,589824=667648,1257472=135168} 2014-07-01 16:16:27,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0435601-112 from 0 to 589824 tid 1 2014-07-01 16:16:27,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0435601-112 from 589824 to 1257472 tid 2 2014-07-01 16:16:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0435601-112 from 1257472 to 1392640 tid 3 2014-07-01 16:16:27,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: write initial oid 113 2014-07-01 16:16:27,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: seq_num 112 ranges {0=458752,458752=651264,1110016=151552} 2014-07-01 16:16:27,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0435601-113 from 0 to 458752 tid 1 2014-07-01 16:16:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0435601-113 from 458752 to 1110016 tid 2 2014-07-01 16:16:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0435601-113 from 1110016 to 1261568 tid 3 2014-07-01 16:16:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: write initial oid 114 2014-07-01 16:16:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: seq_num 113 ranges {0=536576,536576=401408} 2014-07-01 16:16:27,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: writing mira0435601-114 from 0 to 536576 tid 1 2014-07-01 16:16:27,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: writing mira0435601-114 from 536576 to 937984 tid 2 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 5 to mira0435601-99 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 99 v 16 (ObjNum 98 snap 0 seq_num 98) dirty exists 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 6 to mira0435601-104 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 104 v 31 (ObjNum 103 snap 0 seq_num 103) dirty exists 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 5 to mira0435601-106 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 40 (ObjNum 105 snap 0 seq_num 105) dirty exists 2014-07-01 16:16:27,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 7 to mira0435601-85 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 85 v 30 (ObjNum 84 snap 0 seq_num 84) dirty exists 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: done (15 left) 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: done (14 left) 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: done (13 left) 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: done (12 left) 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: write initial oid 115 2014-07-01 16:16:27,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: seq_num 114 ranges {0=409600,409600=401408} 2014-07-01 16:16:27,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0435601-115 from 0 to 409600 tid 1 2014-07-01 16:16:27,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0435601-115 from 409600 to 811008 tid 2 2014-07-01 16:16:27,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: write initial oid 116 2014-07-01 16:16:27,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: seq_num 115 ranges {0=679936} 2014-07-01 16:16:27,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: writing mira0435601-116 from 0 to 679936 tid 1 2014-07-01 16:16:27,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: write initial oid 117 2014-07-01 16:16:27,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: seq_num 116 ranges {0=708608,708608=475136,1183744=475136,1658880=253952} 2014-07-01 16:16:27,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: writing mira0435601-117 from 0 to 708608 tid 1 2014-07-01 16:16:27,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: writing mira0435601-117 from 708608 to 1183744 tid 2 2014-07-01 16:16:27,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: writing mira0435601-117 from 1183744 to 1658880 tid 3 2014-07-01 16:16:27,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: writing mira0435601-117 from 1658880 to 1912832 tid 4 2014-07-01 16:16:27,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: write initial oid 118 2014-07-01 16:16:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: seq_num 117 ranges {0=581632,581632=659456,1241088=544768} 2014-07-01 16:16:27,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: writing mira0435601-118 from 0 to 581632 tid 1 2014-07-01 16:16:27,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: writing mira0435601-118 from 581632 to 1241088 tid 2 2014-07-01 16:16:27,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: writing mira0435601-118 from 1241088 to 1785856 tid 3 2014-07-01 16:16:27,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 6 to mira0435601-103 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 35 (ObjNum 102 snap 0 seq_num 102) dirty exists 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 2 to mira0435601-109 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 5 to mira0435601-107 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 107 v 38 (ObjNum 106 snap 0 seq_num 106) dirty exists 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 3 to mira0435601-109 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: done (15 left) 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: done (14 left) 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: write initial oid 119 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: seq_num 118 ranges {0=98304} 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: writing mira0435601-119 from 0 to 98304 tid 1 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: write initial oid 120 2014-07-01 16:16:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: seq_num 119 ranges {0=528384,528384=471040,999424=331776} 2014-07-01 16:16:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: writing mira0435601-120 from 0 to 528384 tid 1 2014-07-01 16:16:27,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: writing mira0435601-120 from 528384 to 999424 tid 2 2014-07-01 16:16:27,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: writing mira0435601-120 from 999424 to 1331200 tid 3 2014-07-01 16:16:27,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:27,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 5 to mira0435601-109 2014-07-01 16:16:27,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 1 to mira0435601-113 2014-07-01 16:16:27,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 6 to mira0435601-109 2014-07-01 16:16:27,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 35 (ObjNum 108 snap 0 seq_num 108) dirty exists 2014-07-01 16:16:27,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 1 to mira0435601-110 2014-07-01 16:16:27,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 3 to mira0435601-110 2014-07-01 16:16:28,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: done (15 left) 2014-07-01 16:16:28,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: write initial oid 121 2014-07-01 16:16:28,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: seq_num 120 ranges {0=155648} 2014-07-01 16:16:28,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: writing mira0435601-121 from 0 to 155648 tid 1 2014-07-01 16:16:28,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:28,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 4 to mira0435601-110 2014-07-01 16:16:28,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 27 (ObjNum 109 snap 0 seq_num 109) dirty exists 2014-07-01 16:16:28,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: done (15 left) 2014-07-01 16:16:28,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: write initial oid 122 2014-07-01 16:16:28,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: seq_num 121 ranges {0=671744,671744=401408} 2014-07-01 16:16:28,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: writing mira0435601-122 from 0 to 671744 tid 1 2014-07-01 16:16:28,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: writing mira0435601-122 from 671744 to 1073152 tid 2 2014-07-01 16:16:28,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:28,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 1 to mira0435601-95 2014-07-01 16:16:28,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 2 to mira0435601-95 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 3 to mira0435601-95 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 4 to mira0435601-95 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 6 to mira0435601-95 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 7 to mira0435601-95 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 20 (ObjNum 94 snap 0 seq_num 94) dirty exists 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: done (15 left) 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: write initial oid 123 2014-07-01 16:16:28,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: seq_num 122 ranges {0=749568} 2014-07-01 16:16:28,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: writing mira0435601-123 from 0 to 749568 tid 1 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 2 to mira0435601-108 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 3 to mira0435601-108 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 4 to mira0435601-108 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 2 to mira0435601-113 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 6 to mira0435601-108 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 1 to mira0435601-112 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 2 to mira0435601-112 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 3 to mira0435601-112 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 5 to mira0435601-112 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 7 to mira0435601-108 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 25 (ObjNum 107 snap 0 seq_num 107) dirty exists 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: done (15 left) 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: write initial oid 124 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: seq_num 123 ranges {0=622592} 2014-07-01 16:16:28,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: writing mira0435601-124 from 0 to 622592 tid 1 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 1 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 2 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 3 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 4 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 6 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 6 to mira0435601-112 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 112 v 29 (ObjNum 111 snap 0 seq_num 111) dirty exists 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 7 to mira0435601-96 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 25 (ObjNum 95 snap 0 seq_num 95) dirty exists 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: done (15 left) 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: done (14 left) 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: write initial oid 125 2014-07-01 16:16:28,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: seq_num 124 ranges {0=491520} 2014-07-01 16:16:28,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: writing mira0435601-125 from 0 to 491520 tid 1 2014-07-01 16:16:28,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: write initial oid 126 2014-07-01 16:16:28,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: seq_num 125 ranges {0=520192,520192=462848,983040=741376} 2014-07-01 16:16:28,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: writing mira0435601-126 from 0 to 520192 tid 1 2014-07-01 16:16:28,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: writing mira0435601-126 from 520192 to 983040 tid 2 2014-07-01 16:16:28,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: writing mira0435601-126 from 983040 to 1724416 tid 3 2014-07-01 16:16:28,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:28,266.266 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 22 16 142 126 22.9041 76 0.62797 2.65466 2014-07-01 16:16:29,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 23 16 142 126 21.9082 0 - 2.65466 2014-07-01 16:16:29,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 1 to mira0435601-114 2014-07-01 16:16:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 3 to mira0435601-113 2014-07-01 16:16:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 5 to mira0435601-113 2014-07-01 16:16:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 6 to mira0435601-113 2014-07-01 16:16:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 113 v 8 (ObjNum 112 snap 0 seq_num 112) dirty exists 2014-07-01 16:16:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: done (15 left) 2014-07-01 16:16:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: write initial oid 127 2014-07-01 16:16:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: seq_num 126 ranges {0=794624,794624=450560,1245184=352256} 2014-07-01 16:16:29,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: writing mira0435601-127 from 0 to 794624 tid 1 2014-07-01 16:16:29,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: writing mira0435601-127 from 794624 to 1245184 tid 2 2014-07-01 16:16:29,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: writing mira0435601-127 from 1245184 to 1597440 tid 3 2014-07-01 16:16:29,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:29,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 1 to mira0435601-121 2014-07-01 16:16:29,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 3 to mira0435601-121 2014-07-01 16:16:29,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 4 to mira0435601-121 2014-07-01 16:16:29,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 33 (ObjNum 120 snap 0 seq_num 120) dirty exists 2014-07-01 16:16:29,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: done (15 left) 2014-07-01 16:16:29,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: write initial oid 128 2014-07-01 16:16:29,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: seq_num 127 ranges {0=712704,712704=679936,1392640=696320,2088960=229376} 2014-07-01 16:16:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0435601-128 from 0 to 712704 tid 1 2014-07-01 16:16:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0435601-128 from 712704 to 1392640 tid 2 2014-07-01 16:16:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0435601-128 from 1392640 to 2088960 tid 3 2014-07-01 16:16:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0435601-128 from 2088960 to 2318336 tid 4 2014-07-01 16:16:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:30,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 1 to mira0435601-125 2014-07-01 16:16:30,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 3 to mira0435601-125 2014-07-01 16:16:30,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 1 to mira0435601-126 2014-07-01 16:16:30,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 2 to mira0435601-126 2014-07-01 16:16:30,265.265 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 24 16 145 129 21.4953 6 6.08861 2.7037 2014-07-01 16:16:30,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 2 to mira0435601-114 2014-07-01 16:16:30,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 1 to mira0435601-105 2014-07-01 16:16:30,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 4 to mira0435601-114 2014-07-01 16:16:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 2 to mira0435601-105 2014-07-01 16:16:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 1 to mira0435601-118 2014-07-01 16:16:30,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 2 to mira0435601-118 2014-07-01 16:16:30,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 3 to mira0435601-105 2014-07-01 16:16:30,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 4 to mira0435601-105 2014-07-01 16:16:30,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 3 to mira0435601-118 2014-07-01 16:16:30,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 6 to mira0435601-105 2014-07-01 16:16:30,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 5 to mira0435601-118 2014-07-01 16:16:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 1 to mira0435601-115 2014-07-01 16:16:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 2 to mira0435601-115 2014-07-01 16:16:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 4 to mira0435601-115 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 4 to mira0435601-125 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 37 (ObjNum 124 snap 0 seq_num 124) dirty exists 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: done (15 left) 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: write initial oid 129 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: seq_num 128 ranges {0=741376,741376=401408} 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: writing mira0435601-129 from 0 to 741376 tid 1 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: writing mira0435601-129 from 741376 to 1142784 tid 2 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 5 to mira0435601-115 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 14 (ObjNum 114 snap 0 seq_num 114) dirty exists 2014-07-01 16:16:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 7 to mira0435601-105 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 11 (ObjNum 104 snap 0 seq_num 104) dirty exists 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 6 to mira0435601-118 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 32 (ObjNum 117 snap 0 seq_num 117) dirty exists 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 5 to mira0435601-114 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 28 (ObjNum 113 snap 0 seq_num 113) dirty exists 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 3 to mira0435601-126 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 1 to mira0435601-111 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 2 to mira0435601-111 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 3 to mira0435601-111 2014-07-01 16:16:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: done (15 left) 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: done (14 left) 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: done (13 left) 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: done (12 left) 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: write initial oid 130 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: seq_num 129 ranges {0=610304,610304=401408} 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: writing mira0435601-130 from 0 to 610304 tid 1 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: writing mira0435601-130 from 610304 to 1011712 tid 2 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: write initial oid 131 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: seq_num 130 ranges {0=483328,483328=401408} 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0435601-131 from 0 to 483328 tid 1 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0435601-131 from 483328 to 884736 tid 2 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: write initial oid 132 2014-07-01 16:16:30,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: seq_num 131 ranges {0=561152} 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: writing mira0435601-132 from 0 to 561152 tid 1 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: write initial oid 133 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: seq_num 132 ranges {0=430080} 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: writing mira0435601-133 from 0 to 430080 tid 1 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 5 to mira0435601-111 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 1 to mira0435601-119 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 3 to mira0435601-119 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 1 to mira0435601-124 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 3 to mira0435601-124 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 6 to mira0435601-111 2014-07-01 16:16:30,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 16 (ObjNum 110 snap 0 seq_num 110) dirty exists 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 5 to mira0435601-126 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 6 to mira0435601-126 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 33 (ObjNum 125 snap 0 seq_num 125) dirty exists 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 1 to mira0435601-128 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 2 to mira0435601-128 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 3 to mira0435601-128 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 4 to mira0435601-128 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: done (15 left) 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: done (14 left) 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: write initial oid 134 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: seq_num 133 ranges {0=458752,458752=425984,884736=536576,1421312=241664} 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: writing mira0435601-134 from 0 to 458752 tid 1 2014-07-01 16:16:30,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: writing mira0435601-134 from 458752 to 884736 tid 2 2014-07-01 16:16:30,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: writing mira0435601-134 from 884736 to 1421312 tid 3 2014-07-01 16:16:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: writing mira0435601-134 from 1421312 to 1662976 tid 4 2014-07-01 16:16:30,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: write initial oid 135 2014-07-01 16:16:30,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: seq_num 134 ranges {0=733184,733184=409600,1142784=393216} 2014-07-01 16:16:30,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: writing mira0435601-135 from 0 to 733184 tid 1 2014-07-01 16:16:30,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: writing mira0435601-135 from 733184 to 1142784 tid 2 2014-07-01 16:16:30,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: writing mira0435601-135 from 1142784 to 1536000 tid 3 2014-07-01 16:16:30,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:30,985.985 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:30,985.985 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:30,985.985 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 3 to 0.948275672147 2014-07-01 16:16:30,985.985 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 0.948275672147' 2014-07-01 16:16:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 4 to mira0435601-119 2014-07-01 16:16:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 17 (ObjNum 118 snap 0 seq_num 118) dirty exists 2014-07-01 16:16:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: done (15 left) 2014-07-01 16:16:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: write initial oid 136 2014-07-01 16:16:31,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: seq_num 135 ranges {0=602112,602112=483328,1085440=319488} 2014-07-01 16:16:31,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: writing mira0435601-136 from 0 to 602112 tid 1 2014-07-01 16:16:31,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: writing mira0435601-136 from 602112 to 1085440 tid 2 2014-07-01 16:16:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: writing mira0435601-136 from 1085440 to 1404928 tid 3 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 6 to mira0435601-128 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 4 to mira0435601-124 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 19 (ObjNum 123 snap 0 seq_num 123) dirty exists 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 1 to mira0435601-116 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 3 to mira0435601-116 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 1 to mira0435601-117 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 2 to mira0435601-117 2014-07-01 16:16:31,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 3 to mira0435601-117 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 4 to mira0435601-117 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 1 to mira0435601-120 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 6 to mira0435601-117 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 2 to mira0435601-120 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 3 to mira0435601-120 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 1 to mira0435601-122 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 5 to mira0435601-120 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 1 to mira0435601-127 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 2 to mira0435601-127 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 3 to mira0435601-127 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 5 to mira0435601-127 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: done (15 left) 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: write initial oid 137 2014-07-01 16:16:31,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: seq_num 136 ranges {0=679936,679936=401408} 2014-07-01 16:16:31,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: writing mira0435601-137 from 0 to 679936 tid 1 2014-07-01 16:16:31,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: writing mira0435601-137 from 679936 to 1081344 tid 2 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 2 to mira0435601-122 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 4 to mira0435601-122 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 1 to mira0435601-123 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 3 to mira0435601-123 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 6 to mira0435601-127 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 23 (ObjNum 126 snap 0 seq_num 126) dirty exists 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: done (15 left) 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: write initial oid 138 2014-07-01 16:16:31,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: seq_num 137 ranges {0=552960,552960=401408} 2014-07-01 16:16:31,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: writing mira0435601-138 from 0 to 552960 tid 1 2014-07-01 16:16:31,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: writing mira0435601-138 from 552960 to 954368 tid 2 2014-07-01 16:16:31,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:31,267.267 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 25 16 160 144 23.0349 60 0.965294 2.72373 2014-07-01 16:16:31,782.782 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 0.948276 (8621462) 2014-07-01 16:16:32,267.267 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 26 16 161 145 22.3027 4 1.5321 2.71551 2014-07-01 16:16:32,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 4 to mira0435601-116 2014-07-01 16:16:32,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 31 (ObjNum 115 snap 0 seq_num 115) dirty exists 2014-07-01 16:16:32,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: done (15 left) 2014-07-01 16:16:32,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: write initial oid 139 2014-07-01 16:16:32,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: seq_num 138 ranges {0=421888,421888=401408} 2014-07-01 16:16:32,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: writing mira0435601-139 from 0 to 421888 tid 1 2014-07-01 16:16:32,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: writing mira0435601-139 from 421888 to 823296 tid 2 2014-07-01 16:16:32,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:32,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 6 to mira0435601-120 2014-07-01 16:16:32,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 35 (ObjNum 119 snap 0 seq_num 119) dirty exists 2014-07-01 16:16:32,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 7 to mira0435601-128 2014-07-01 16:16:32,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 37 (ObjNum 127 snap 0 seq_num 127) dirty exists 2014-07-01 16:16:32,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: done (15 left) 2014-07-01 16:16:32,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: done (14 left) 2014-07-01 16:16:32,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: write initial oid 140 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: seq_num 139 ranges {0=450560,450560=466944,917504=483328,1400832=622592,2023424=32768} 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0435601-140 from 0 to 450560 tid 1 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0435601-140 from 450560 to 917504 tid 2 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0435601-140 from 917504 to 1400832 tid 3 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0435601-140 from 1400832 to 2023424 tid 4 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0435601-140 from 2023424 to 2056192 tid 5 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: write initial oid 141 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: seq_num 140 ranges {0=372736} 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: writing mira0435601-141 from 0 to 372736 tid 1 2014-07-01 16:16:32,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:33,267.267 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 27 16 164 148 21.921 12 1.65806 2.69268 2014-07-01 16:16:34,267.267 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 28 16 164 148 21.1381 0 - 2.69268 2014-07-01 16:16:35,268.268 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 29 16 166 150 20.685 4 1.91343 2.68223 2014-07-01 16:16:36,268.268 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 30 16 166 150 19.9955 0 - 2.68223 2014-07-01 16:16:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 1 to mira0435601-130 2014-07-01 16:16:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 2 to mira0435601-130 2014-07-01 16:16:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 4 to mira0435601-130 2014-07-01 16:16:36,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 1 to mira0435601-133 2014-07-01 16:16:36,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 3 to mira0435601-133 2014-07-01 16:16:36,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 1 to mira0435601-135 2014-07-01 16:16:36,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 2 to mira0435601-135 2014-07-01 16:16:36,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 3 to mira0435601-135 2014-07-01 16:16:36,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 5 to mira0435601-135 2014-07-01 16:16:36,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 5 to mira0435601-130 2014-07-01 16:16:36,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 41 (ObjNum 129 snap 0 seq_num 129) dirty exists 2014-07-01 16:16:36,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: done (15 left) 2014-07-01 16:16:36,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: write initial oid 142 2014-07-01 16:16:36,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: seq_num 141 ranges {0=241664} 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: writing mira0435601-142 from 0 to 241664 tid 1 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 4 to mira0435601-133 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 43 (ObjNum 132 snap 0 seq_num 132) dirty exists 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: done (15 left) 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: write initial oid 143 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: seq_num 142 ranges {0=671744,671744=618496,1290240=184320} 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: writing mira0435601-143 from 0 to 671744 tid 1 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: writing mira0435601-143 from 671744 to 1290240 tid 2 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: writing mira0435601-143 from 1290240 to 1474560 tid 3 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 1 to mira0435601-131 2014-07-01 16:16:36,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 1 to mira0435601-129 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 2 to mira0435601-131 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 1 to mira0435601-132 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 2 to mira0435601-129 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 3 to mira0435601-132 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 4 to mira0435601-131 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 4 to mira0435601-129 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 1 to mira0435601-134 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 2 to mira0435601-134 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 3 to mira0435601-134 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 4 to mira0435601-134 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 6 to mira0435601-134 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 7 to mira0435601-117 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 117 v 32 (ObjNum 116 snap 0 seq_num 116) dirty exists 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: done (15 left) 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: write initial oid 144 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: seq_num 143 ranges {0=299008} 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: writing mira0435601-144 from 0 to 299008 tid 1 2014-07-01 16:16:36,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 5 to mira0435601-122 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 35 (ObjNum 121 snap 0 seq_num 121) dirty exists 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 4 to mira0435601-123 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 37 (ObjNum 122 snap 0 seq_num 122) dirty exists 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 5 to mira0435601-131 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 11 (ObjNum 130 snap 0 seq_num 130) dirty exists 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 6 to mira0435601-135 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 135 v 18 (ObjNum 134 snap 0 seq_num 134) dirty exists 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 5 to mira0435601-129 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 40 (ObjNum 128 snap 0 seq_num 128) dirty exists 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: done (15 left) 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: done (14 left) 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: done (13 left) 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: done (12 left) 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: done (11 left) 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: write initial oid 145 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: seq_num 144 ranges {0=413696,413696=430080,843776=372736} 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0435601-145 from 0 to 413696 tid 1 2014-07-01 16:16:36,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0435601-145 from 413696 to 843776 tid 2 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0435601-145 from 843776 to 1216512 tid 3 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: write initial oid 146 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: seq_num 145 ranges {0=491520,491520=401408} 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: writing mira0435601-146 from 0 to 491520 tid 1 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: writing mira0435601-146 from 491520 to 892928 tid 2 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: write initial oid 147 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: seq_num 146 ranges {0=520192,520192=442368,962560=704512,1667072=425984,2093056=32768} 2014-07-01 16:16:36,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0435601-147 from 0 to 520192 tid 1 2014-07-01 16:16:36,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0435601-147 from 520192 to 962560 tid 2 2014-07-01 16:16:36,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0435601-147 from 962560 to 1667072 tid 3 2014-07-01 16:16:36,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0435601-147 from 1667072 to 2093056 tid 4 2014-07-01 16:16:36,795.795 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:36,795.795 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:36,795.795 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 2 to 0.694502066176 2014-07-01 16:16:36,795.795 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 0.694502066176' 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0435601-147 from 2093056 to 2125824 tid 5 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: write initial oid 148 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 4 to mira0435601-132 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 132 v 18 (ObjNum 131 snap 0 seq_num 131) dirty exists 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 1 to mira0435601-136 2014-07-01 16:16:36,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: seq_num 147 ranges {0=634880} 2014-07-01 16:16:36,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: writing mira0435601-148 from 0 to 634880 tid 1 2014-07-01 16:16:36,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: done (14 left) 2014-07-01 16:16:36,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: write initial oid 149 2014-07-01 16:16:36,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: seq_num 148 ranges {0=663552,663552=659456,1323008=544768} 2014-07-01 16:16:36,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: writing mira0435601-149 from 0 to 663552 tid 1 2014-07-01 16:16:36,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: writing mira0435601-149 from 663552 to 1323008 tid 2 2014-07-01 16:16:36,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: writing mira0435601-149 from 1323008 to 1867776 tid 3 2014-07-01 16:16:36,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: write initial oid 150 2014-07-01 16:16:36,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: seq_num 149 ranges {0=737280,737280=487424,1224704=315392} 2014-07-01 16:16:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: writing mira0435601-150 from 0 to 737280 tid 1 2014-07-01 16:16:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: writing mira0435601-150 from 737280 to 1224704 tid 2 2014-07-01 16:16:36,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: writing mira0435601-150 from 1224704 to 1540096 tid 3 2014-07-01 16:16:36,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:36,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 2 to mira0435601-136 2014-07-01 16:16:36,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 1 to mira0435601-138 2014-07-01 16:16:36,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 3 to mira0435601-136 2014-07-01 16:16:36,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 5 to mira0435601-136 2014-07-01 16:16:36,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 2 to mira0435601-138 2014-07-01 16:16:36,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 4 to mira0435601-138 2014-07-01 16:16:36,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 1 to mira0435601-137 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 2 to mira0435601-137 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 4 to mira0435601-137 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 5 to mira0435601-138 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 138 v 21 (ObjNum 137 snap 0 seq_num 137) dirty exists 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 5 to mira0435601-137 2014-07-01 16:16:36,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 26 (ObjNum 136 snap 0 seq_num 136) dirty exists 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 7 to mira0435601-134 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 35 (ObjNum 133 snap 0 seq_num 133) dirty exists 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 1 to mira0435601-140 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 2 to mira0435601-140 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: done (15 left) 2014-07-01 16:16:36,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: done (14 left) 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: done (13 left) 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: write initial oid 151 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: seq_num 150 ranges {0=53248} 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: writing mira0435601-151 from 0 to 53248 tid 1 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: write initial oid 152 2014-07-01 16:16:36,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: seq_num 151 ranges {0=483328,483328=655360,1138688=147456} 2014-07-01 16:16:36,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: writing mira0435601-152 from 0 to 483328 tid 1 2014-07-01 16:16:36,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: writing mira0435601-152 from 483328 to 1138688 tid 2 2014-07-01 16:16:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: writing mira0435601-152 from 1138688 to 1286144 tid 3 2014-07-01 16:16:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: write initial oid 153 2014-07-01 16:16:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: seq_num 152 ranges {0=106496} 2014-07-01 16:16:36,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: writing mira0435601-153 from 0 to 106496 tid 1 2014-07-01 16:16:36,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:36,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 1 to mira0435601-139 2014-07-01 16:16:36,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 2 to mira0435601-139 2014-07-01 16:16:36,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 4 to mira0435601-139 2014-07-01 16:16:36,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 3 to mira0435601-140 2014-07-01 16:16:36,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 4 to mira0435601-140 2014-07-01 16:16:36,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 5 to mira0435601-140 2014-07-01 16:16:36,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 7 to mira0435601-140 2014-07-01 16:16:36,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 5 to mira0435601-139 2014-07-01 16:16:36,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 139 v 46 (ObjNum 138 snap 0 seq_num 138) dirty exists 2014-07-01 16:16:36,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 8 to mira0435601-140 2014-07-01 16:16:36,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 27 (ObjNum 139 snap 0 seq_num 139) dirty exists 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 1 to mira0435601-142 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 1 to mira0435601-144 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 3 to mira0435601-144 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 1 to mira0435601-145 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 2 to mira0435601-145 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 3 to mira0435601-145 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 5 to mira0435601-145 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 4 to mira0435601-144 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 29 (ObjNum 143 snap 0 seq_num 143) dirty exists 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 6 to mira0435601-145 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 33 (ObjNum 144 snap 0 seq_num 144) dirty exists 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 1 to mira0435601-141 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 3 to mira0435601-142 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 3 to mira0435601-141 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 1 to mira0435601-143 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: done (15 left) 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: done (14 left) 2014-07-01 16:16:36,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: done (13 left) 2014-07-01 16:16:36,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: done (12 left) 2014-07-01 16:16:36,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: write initial oid 154 2014-07-01 16:16:36,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: seq_num 153 ranges {0=626688,626688=401408} 2014-07-01 16:16:36,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: writing mira0435601-154 from 0 to 626688 tid 1 2014-07-01 16:16:36,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: writing mira0435601-154 from 626688 to 1028096 tid 2 2014-07-01 16:16:36,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: write initial oid 155 2014-07-01 16:16:36,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: seq_num 154 ranges {0=704512} 2014-07-01 16:16:36,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: writing mira0435601-155 from 0 to 704512 tid 1 2014-07-01 16:16:37,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: write initial oid 156 2014-07-01 16:16:37,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: seq_num 155 ranges {0=729088,729088=634880,1363968=421888,1785856=147456} 2014-07-01 16:16:37,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0435601-156 from 0 to 729088 tid 1 2014-07-01 16:16:37,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0435601-156 from 729088 to 1363968 tid 2 2014-07-01 16:16:37,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0435601-156 from 1363968 to 1785856 tid 3 2014-07-01 16:16:37,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0435601-156 from 1785856 to 1933312 tid 4 2014-07-01 16:16:37,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: write initial oid 157 2014-07-01 16:16:37,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: seq_num 156 ranges {0=602112,602112=462848,1064960=700416,1765376=40960} 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0435601-157 from 0 to 602112 tid 1 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0435601-157 from 602112 to 1064960 tid 2 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0435601-157 from 1064960 to 1765376 tid 3 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0435601-157 from 1765376 to 1806336 tid 4 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 4 to mira0435601-141 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 17 (ObjNum 140 snap 0 seq_num 140) dirty exists 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 6 to mira0435601-136 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 15 (ObjNum 135 snap 0 seq_num 135) dirty exists 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 2 to mira0435601-143 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 3 to mira0435601-143 2014-07-01 16:16:37,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 5 to mira0435601-143 2014-07-01 16:16:37,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 1 to mira0435601-147 2014-07-01 16:16:37,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 2 to mira0435601-147 2014-07-01 16:16:37,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 1 to mira0435601-146 2014-07-01 16:16:37,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 4 to mira0435601-142 2014-07-01 16:16:37,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 142 v 39 (ObjNum 141 snap 0 seq_num 141) dirty exists 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: done (15 left) 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: done (14 left) 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: done (13 left) 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: write initial oid 158 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: seq_num 157 ranges {0=475136,475136=692224,1167360=512000} 2014-07-01 16:16:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: writing mira0435601-158 from 0 to 475136 tid 1 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: writing mira0435601-158 from 475136 to 1167360 tid 2 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: writing mira0435601-158 from 1167360 to 1679360 tid 3 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: write initial oid 159 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: seq_num 158 ranges {0=548864,548864=630784,1179648=172032} 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: writing mira0435601-159 from 0 to 548864 tid 1 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: writing mira0435601-159 from 548864 to 1179648 tid 2 2014-07-01 16:16:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: writing mira0435601-159 from 1179648 to 1351680 tid 3 2014-07-01 16:16:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: write initial oid 160 2014-07-01 16:16:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: seq_num 159 ranges {0=421888,421888=458752,880640=344064} 2014-07-01 16:16:37,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: writing mira0435601-160 from 0 to 421888 tid 1 2014-07-01 16:16:37,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: writing mira0435601-160 from 421888 to 880640 tid 2 2014-07-01 16:16:37,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: writing mira0435601-160 from 880640 to 1224704 tid 3 2014-07-01 16:16:37,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,268.268 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 31 16 183 167 21.5436 34 0.327157 2.91612 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 1 to mira0435601-150 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 2 to mira0435601-150 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 3 to mira0435601-150 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 5 to mira0435601-150 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 1 to mira0435601-152 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 2 to mira0435601-152 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 6 to mira0435601-150 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 150 v 22 (ObjNum 149 snap 0 seq_num 149) dirty exists 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: done (15 left) 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: write initial oid 161 2014-07-01 16:16:37,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: seq_num 160 ranges {0=696320,696320=401408} 2014-07-01 16:16:37,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: writing mira0435601-161 from 0 to 696320 tid 1 2014-07-01 16:16:37,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: writing mira0435601-161 from 696320 to 1097728 tid 2 2014-07-01 16:16:37,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 1 to mira0435601-156 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 1 to mira0435601-151 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 3 to mira0435601-151 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 1 to mira0435601-154 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 2 to mira0435601-154 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 4 to mira0435601-154 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 2 to mira0435601-156 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 3 to mira0435601-156 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 4 to mira0435601-156 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 6 to mira0435601-156 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 4 to mira0435601-151 2014-07-01 16:16:37,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 35 (ObjNum 150 snap 0 seq_num 150) dirty exists 2014-07-01 16:16:37,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 5 to mira0435601-154 2014-07-01 16:16:37,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 38 (ObjNum 153 snap 0 seq_num 153) dirty exists 2014-07-01 16:16:37,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: done (15 left) 2014-07-01 16:16:37,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: done (14 left) 2014-07-01 16:16:37,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: write initial oid 162 2014-07-01 16:16:37,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: seq_num 161 ranges {0=720896,720896=675840,1396736=663552,2060288=266240} 2014-07-01 16:16:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0435601-162 from 0 to 720896 tid 1 2014-07-01 16:16:37,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0435601-162 from 720896 to 1396736 tid 2 2014-07-01 16:16:37,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0435601-162 from 1396736 to 2060288 tid 3 2014-07-01 16:16:37,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0435601-162 from 2060288 to 2326528 tid 4 2014-07-01 16:16:37,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: write initial oid 163 2014-07-01 16:16:37,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: seq_num 162 ranges {0=593920,593920=610304,1204224=491520,1695744=503808} 2014-07-01 16:16:37,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0435601-163 from 0 to 593920 tid 1 2014-07-01 16:16:37,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0435601-163 from 593920 to 1204224 tid 2 2014-07-01 16:16:37,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0435601-163 from 1204224 to 1695744 tid 3 2014-07-01 16:16:37,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0435601-163 from 1695744 to 2199552 tid 4 2014-07-01 16:16:37,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 7 to mira0435601-156 2014-07-01 16:16:37,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 31 (ObjNum 155 snap 0 seq_num 155) dirty exists 2014-07-01 16:16:37,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 1 to mira0435601-148 2014-07-01 16:16:37,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 3 to mira0435601-148 2014-07-01 16:16:37,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 6 to mira0435601-143 2014-07-01 16:16:37,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 39 (ObjNum 142 snap 0 seq_num 142) dirty exists 2014-07-01 16:16:37,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 4 to mira0435601-148 2014-07-01 16:16:37,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 37 (ObjNum 147 snap 0 seq_num 147) dirty exists 2014-07-01 16:16:37,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 1 to mira0435601-159 2014-07-01 16:16:37,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 2 to mira0435601-159 2014-07-01 16:16:37,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 3 to mira0435601-159 2014-07-01 16:16:37,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 5 to mira0435601-159 2014-07-01 16:16:37,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 6 to mira0435601-159 2014-07-01 16:16:37,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 159 v 50 (ObjNum 158 snap 0 seq_num 158) dirty exists 2014-07-01 16:16:37,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: done (15 left) 2014-07-01 16:16:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: done (14 left) 2014-07-01 16:16:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: done (13 left) 2014-07-01 16:16:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: done (12 left) 2014-07-01 16:16:37,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: write initial oid 164 2014-07-01 16:16:37,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: seq_num 163 ranges {0=516096} 2014-07-01 16:16:37,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: writing mira0435601-164 from 0 to 516096 tid 1 2014-07-01 16:16:37,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: write initial oid 165 2014-07-01 16:16:37,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: seq_num 164 ranges {0=540672,540672=425984,966656=438272,1404928=339968} 2014-07-01 16:16:37,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0435601-165 from 0 to 540672 tid 1 2014-07-01 16:16:37,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0435601-165 from 540672 to 966656 tid 2 2014-07-01 16:16:37,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0435601-165 from 966656 to 1404928 tid 3 2014-07-01 16:16:37,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0435601-165 from 1404928 to 1744896 tid 4 2014-07-01 16:16:37,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: write initial oid 166 2014-07-01 16:16:37,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: seq_num 165 ranges {0=413696,413696=655360,1069056=548864} 2014-07-01 16:16:37,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: writing mira0435601-166 from 0 to 413696 tid 1 2014-07-01 16:16:37,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: writing mira0435601-166 from 413696 to 1069056 tid 2 2014-07-01 16:16:37,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: writing mira0435601-166 from 1069056 to 1617920 tid 3 2014-07-01 16:16:37,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: write initial oid 167 2014-07-01 16:16:37,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: seq_num 166 ranges {0=688128,688128=589824,1277952=212992} 2014-07-01 16:16:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: writing mira0435601-167 from 0 to 688128 tid 1 2014-07-01 16:16:37,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: writing mira0435601-167 from 688128 to 1277952 tid 2 2014-07-01 16:16:37,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: writing mira0435601-167 from 1277952 to 1490944 tid 3 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 2 to mira0435601-146 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 4 to mira0435601-146 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 1 to mira0435601-149 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 2 to mira0435601-149 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 3 to mira0435601-149 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 1 to mira0435601-153 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 5 to mira0435601-149 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 3 to mira0435601-153 2014-07-01 16:16:37,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 1 to mira0435601-158 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 3 to mira0435601-147 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 4 to mira0435601-147 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 5 to mira0435601-146 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 19 (ObjNum 145 snap 0 seq_num 145) dirty exists 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 5 to mira0435601-147 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 7 to mira0435601-147 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 6 to mira0435601-149 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 27 (ObjNum 148 snap 0 seq_num 148) dirty exists 2014-07-01 16:16:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 4 to mira0435601-153 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 21 (ObjNum 152 snap 0 seq_num 152) dirty exists 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 8 to mira0435601-147 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 45 (ObjNum 146 snap 0 seq_num 146) dirty exists 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 2 to mira0435601-158 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 3 to mira0435601-158 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 5 to mira0435601-158 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 1 to mira0435601-160 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 2 to mira0435601-160 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 3 to mira0435601-160 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 5 to mira0435601-160 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 6 to mira0435601-160 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 35 (ObjNum 159 snap 0 seq_num 159) dirty exists 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 1 to mira0435601-161 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 2 to mira0435601-161 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 4 to mira0435601-161 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 5 to mira0435601-161 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 42 (ObjNum 160 snap 0 seq_num 160) dirty exists 2014-07-01 16:16:37,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 1 to mira0435601-162 2014-07-01 16:16:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 2 to mira0435601-162 2014-07-01 16:16:37,671.671 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 0.694502 (8455142) 2014-07-01 16:16:37,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: done (15 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: done (14 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: done (13 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: done (12 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: done (11 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: done (10 left) 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: write initial oid 168 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: seq_num 167 ranges {0=606208,606208=667648,1273856=610304,1884160=327680} 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0435601-168 from 0 to 606208 tid 1 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0435601-168 from 606208 to 1273856 tid 2 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0435601-168 from 1273856 to 1884160 tid 3 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0435601-168 from 1884160 to 2211840 tid 4 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: write initial oid 169 2014-07-01 16:16:37,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: seq_num 168 ranges {0=634880,634880=401408} 2014-07-01 16:16:37,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: writing mira0435601-169 from 0 to 634880 tid 1 2014-07-01 16:16:37,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: writing mira0435601-169 from 634880 to 1036288 tid 2 2014-07-01 16:16:37,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: write initial oid 170 2014-07-01 16:16:37,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: seq_num 169 ranges {0=663552,663552=634880,1298432=557056,1855488=413696} 2014-07-01 16:16:37,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0435601-170 from 0 to 663552 tid 1 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0435601-170 from 663552 to 1298432 tid 2 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0435601-170 from 1298432 to 1855488 tid 3 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0435601-170 from 1855488 to 2269184 tid 4 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: write initial oid 171 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: seq_num 170 ranges {0=778240} 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: writing mira0435601-171 from 0 to 778240 tid 1 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: write initial oid 172 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: seq_num 171 ranges {0=405504,405504=802816,1208320=659456,1867776=143360} 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0435601-172 from 0 to 405504 tid 1 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0435601-172 from 405504 to 1208320 tid 2 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0435601-172 from 1208320 to 1867776 tid 3 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0435601-172 from 1867776 to 2011136 tid 4 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: write initial oid 173 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: seq_num 172 ranges {0=479232,479232=630784,1110016=573440} 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0435601-173 from 0 to 479232 tid 1 2014-07-01 16:16:37,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0435601-173 from 479232 to 1110016 tid 2 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0435601-173 from 1110016 to 1683456 tid 3 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 6 to mira0435601-158 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 158 v 25 (ObjNum 157 snap 0 seq_num 157) dirty exists 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: done (15 left) 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: write initial oid 174 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: seq_num 173 ranges {0=196608} 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: writing mira0435601-174 from 0 to 196608 tid 1 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 1 to mira0435601-164 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 3 to mira0435601-164 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 4 to mira0435601-164 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 35 (ObjNum 163 snap 0 seq_num 163) dirty exists 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: done (15 left) 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: write initial oid 175 2014-07-01 16:16:37,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: seq_num 174 ranges {0=626688,626688=798720,1425408=4096} 2014-07-01 16:16:37,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: writing mira0435601-175 from 0 to 626688 tid 1 2014-07-01 16:16:37,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: writing mira0435601-175 from 626688 to 1425408 tid 2 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: writing mira0435601-175 from 1425408 to 1429504 tid 3 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 1 to mira0435601-171 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 3 to mira0435601-171 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 1 to mira0435601-172 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 3 to mira0435601-162 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 4 to mira0435601-162 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 2 to mira0435601-172 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 3 to mira0435601-172 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 4 to mira0435601-172 2014-07-01 16:16:37,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 6 to mira0435601-172 2014-07-01 16:16:38,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 6 to mira0435601-162 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 1 to mira0435601-163 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 2 to mira0435601-163 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 3 to mira0435601-163 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 1 to mira0435601-165 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 2 to mira0435601-165 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 3 to mira0435601-165 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 4 to mira0435601-165 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 4 to mira0435601-163 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 6 to mira0435601-165 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 1 to mira0435601-168 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 6 to mira0435601-163 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 2 to mira0435601-168 2014-07-01 16:16:38,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 1 to mira0435601-167 2014-07-01 16:16:38,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 2 to mira0435601-167 2014-07-01 16:16:38,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 3 to mira0435601-167 2014-07-01 16:16:38,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 5 to mira0435601-167 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 6 to mira0435601-167 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 167 v 37 (ObjNum 166 snap 0 seq_num 166) dirty exists 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: done (15 left) 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: write initial oid 176 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: seq_num 175 ranges {0=249856} 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: writing mira0435601-176 from 0 to 249856 tid 1 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 7 to mira0435601-162 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 45 (ObjNum 161 snap 0 seq_num 161) dirty exists 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 7 to mira0435601-163 2014-07-01 16:16:38,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 42 (ObjNum 162 snap 0 seq_num 162) dirty exists 2014-07-01 16:16:38,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 3 to mira0435601-168 2014-07-01 16:16:38,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 4 to mira0435601-168 2014-07-01 16:16:38,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 6 to mira0435601-168 2014-07-01 16:16:38,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 7 to mira0435601-165 2014-07-01 16:16:38,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 40 (ObjNum 164 snap 0 seq_num 164) dirty exists 2014-07-01 16:16:38,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: done (15 left) 2014-07-01 16:16:38,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: done (14 left) 2014-07-01 16:16:38,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: done (13 left) 2014-07-01 16:16:38,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: write initial oid 177 2014-07-01 16:16:38,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: seq_num 176 ranges {0=417792,417792=614400,1032192=778240,1810432=212992} 2014-07-01 16:16:38,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0435601-177 from 0 to 417792 tid 1 2014-07-01 16:16:38,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0435601-177 from 417792 to 1032192 tid 2 2014-07-01 16:16:38,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0435601-177 from 1032192 to 1810432 tid 3 2014-07-01 16:16:38,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0435601-177 from 1810432 to 2023424 tid 4 2014-07-01 16:16:38,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: write initial oid 178 2014-07-01 16:16:38,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: seq_num 177 ranges {0=446464,446464=401408} 2014-07-01 16:16:38,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: writing mira0435601-178 from 0 to 446464 tid 1 2014-07-01 16:16:38,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: writing mira0435601-178 from 446464 to 847872 tid 2 2014-07-01 16:16:38,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: write initial oid 179 2014-07-01 16:16:38,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: seq_num 178 ranges {0=471040,471040=622592,1093632=724992,1818624=258048} 2014-07-01 16:16:38,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0435601-179 from 0 to 471040 tid 1 2014-07-01 16:16:38,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0435601-179 from 471040 to 1093632 tid 2 2014-07-01 16:16:38,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0435601-179 from 1093632 to 1818624 tid 3 2014-07-01 16:16:38,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0435601-179 from 1818624 to 2076672 tid 4 2014-07-01 16:16:38,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:38,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 3 to mira0435601-152 2014-07-01 16:16:38,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 5 to mira0435601-152 2014-07-01 16:16:38,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 6 to mira0435601-152 2014-07-01 16:16:38,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 26 (ObjNum 151 snap 0 seq_num 151) dirty exists 2014-07-01 16:16:38,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: done (15 left) 2014-07-01 16:16:38,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: write initial oid 180 2014-07-01 16:16:38,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: seq_num 179 ranges {0=589824} 2014-07-01 16:16:38,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: writing mira0435601-180 from 0 to 589824 tid 1 2014-07-01 16:16:38,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:38,269.269 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 32 16 203 187 23.3698 80 0.579356 2.68938 2014-07-01 16:16:38,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 1 to mira0435601-169 2014-07-01 16:16:38,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 2 to mira0435601-169 2014-07-01 16:16:38,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 4 to mira0435601-169 2014-07-01 16:16:38,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 5 to mira0435601-169 2014-07-01 16:16:38,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 169 v 20 (ObjNum 168 snap 0 seq_num 168) dirty exists 2014-07-01 16:16:38,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: done (15 left) 2014-07-01 16:16:38,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: write initial oid 181 2014-07-01 16:16:38,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: seq_num 180 ranges {0=618496,618496=438272,1056768=425984,1482752=339968} 2014-07-01 16:16:38,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: writing mira0435601-181 from 0 to 618496 tid 1 2014-07-01 16:16:38,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: writing mira0435601-181 from 618496 to 1056768 tid 2 2014-07-01 16:16:38,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: writing mira0435601-181 from 1056768 to 1482752 tid 3 2014-07-01 16:16:38,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: writing mira0435601-181 from 1482752 to 1822720 tid 4 2014-07-01 16:16:38,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:38,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 1 to mira0435601-174 2014-07-01 16:16:38,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 3 to mira0435601-174 2014-07-01 16:16:38,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 4 to mira0435601-174 2014-07-01 16:16:38,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 22 (ObjNum 173 snap 0 seq_num 173) dirty exists 2014-07-01 16:16:38,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: done (15 left) 2014-07-01 16:16:38,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: write initial oid 182 2014-07-01 16:16:38,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: seq_num 181 ranges {0=692224,692224=421888,1114112=380928} 2014-07-01 16:16:38,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: writing mira0435601-182 from 0 to 692224 tid 1 2014-07-01 16:16:38,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: writing mira0435601-182 from 692224 to 1114112 tid 2 2014-07-01 16:16:38,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: writing mira0435601-182 from 1114112 to 1495040 tid 3 2014-07-01 16:16:38,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:39,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 1 to mira0435601-173 2014-07-01 16:16:39,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 2 to mira0435601-173 2014-07-01 16:16:39,268.268 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 33 16 208 192 23.2675 20 1.23726 2.65837 2014-07-01 16:16:39,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 1 to mira0435601-177 2014-07-01 16:16:39,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 2 to mira0435601-177 2014-07-01 16:16:39,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 1 to mira0435601-178 2014-07-01 16:16:39,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 3 to mira0435601-177 2014-07-01 16:16:39,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 4 to mira0435601-177 2014-07-01 16:16:39,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 6 to mira0435601-177 2014-07-01 16:16:39,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 2 to mira0435601-178 2014-07-01 16:16:39,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 4 to mira0435601-178 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 1 to mira0435601-180 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 3 to mira0435601-180 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 7 to mira0435601-177 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 50 (ObjNum 176 snap 0 seq_num 176) dirty exists 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: done (15 left) 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: write initial oid 183 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: seq_num 182 ranges {0=565248,565248=602112,1167360=200704} 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: writing mira0435601-183 from 0 to 565248 tid 1 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: writing mira0435601-183 from 565248 to 1167360 tid 2 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: writing mira0435601-183 from 1167360 to 1368064 tid 3 2014-07-01 16:16:39,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:39,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 1 to mira0435601-176 2014-07-01 16:16:39,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 3 to mira0435601-176 2014-07-01 16:16:39,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 1 to mira0435601-179 2014-07-01 16:16:39,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 2 to mira0435601-179 2014-07-01 16:16:39,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 3 to mira0435601-173 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 3 to mira0435601-179 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 5 to mira0435601-173 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 4 to mira0435601-179 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 1 to mira0435601-175 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 2 to mira0435601-175 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 6 to mira0435601-179 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 3 to mira0435601-175 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 5 to mira0435601-175 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 6 to mira0435601-173 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 30 (ObjNum 172 snap 0 seq_num 172) dirty exists 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: done (15 left) 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: write initial oid 184 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: seq_num 183 ranges {0=438272,438272=589824,1028096=212992} 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: writing mira0435601-184 from 0 to 438272 tid 1 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: writing mira0435601-184 from 438272 to 1028096 tid 2 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: writing mira0435601-184 from 1028096 to 1241088 tid 3 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 1 to mira0435601-181 2014-07-01 16:16:39,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 2 to mira0435601-181 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 3 to mira0435601-181 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 4 to mira0435601-181 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 6 to mira0435601-181 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 6 to mira0435601-175 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 175 v 34 (ObjNum 174 snap 0 seq_num 174) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 7 to mira0435601-168 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 50 (ObjNum 167 snap 0 seq_num 167) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 7 to mira0435601-181 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 47 (ObjNum 180 snap 0 seq_num 180) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 4 to mira0435601-171 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 37 (ObjNum 170 snap 0 seq_num 170) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 7 to mira0435601-172 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 42 (ObjNum 171 snap 0 seq_num 171) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 4 to mira0435601-176 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 44 (ObjNum 175 snap 0 seq_num 175) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 5 to mira0435601-178 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 41 (ObjNum 177 snap 0 seq_num 177) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 7 to mira0435601-179 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 179 v 49 (ObjNum 178 snap 0 seq_num 178) dirty exists 2014-07-01 16:16:39,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 4 to mira0435601-180 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 43 (ObjNum 179 snap 0 seq_num 179) dirty exists 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: done (15 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: done (14 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: done (13 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: done (12 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: done (11 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: done (10 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: done (9 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: done (8 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: done (7 left) 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: write initial oid 185 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: seq_num 184 ranges {0=61440} 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: writing mira0435601-185 from 0 to 61440 tid 1 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: write initial oid 186 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: seq_num 185 ranges {0=786432} 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: writing mira0435601-186 from 0 to 786432 tid 1 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: write initial oid 187 2014-07-01 16:16:39,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: seq_num 186 ranges {0=659456} 2014-07-01 16:16:39,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: writing mira0435601-187 from 0 to 659456 tid 1 2014-07-01 16:16:39,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: write initial oid 188 2014-07-01 16:16:39,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: seq_num 187 ranges {0=684032,684032=569344,1253376=442368,1695744=192512} 2014-07-01 16:16:39,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0435601-188 from 0 to 684032 tid 1 2014-07-01 16:16:39,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0435601-188 from 684032 to 1253376 tid 2 2014-07-01 16:16:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0435601-188 from 1253376 to 1695744 tid 3 2014-07-01 16:16:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0435601-188 from 1695744 to 1888256 tid 4 2014-07-01 16:16:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: write initial oid 189 2014-07-01 16:16:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: seq_num 188 ranges {0=557056,557056=798720,1355776=405504} 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0435601-189 from 0 to 557056 tid 1 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0435601-189 from 557056 to 1355776 tid 2 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0435601-189 from 1355776 to 1761280 tid 3 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: write initial oid 190 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: seq_num 189 ranges {0=430080,430080=630784,1060864=573440} 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: writing mira0435601-190 from 0 to 430080 tid 1 2014-07-01 16:16:39,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: writing mira0435601-190 from 430080 to 1060864 tid 2 2014-07-01 16:16:39,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: writing mira0435601-190 from 1060864 to 1634304 tid 3 2014-07-01 16:16:39,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: write initial oid 191 2014-07-01 16:16:39,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: seq_num 190 ranges {0=503808,503808=409600,913408=393216} 2014-07-01 16:16:39,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: writing mira0435601-191 from 0 to 503808 tid 1 2014-07-01 16:16:39,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: writing mira0435601-191 from 503808 to 913408 tid 2 2014-07-01 16:16:39,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: writing mira0435601-191 from 913408 to 1306624 tid 3 2014-07-01 16:16:39,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: write initial oid 192 2014-07-01 16:16:39,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: seq_num 191 ranges {0=778240,778240=401408} 2014-07-01 16:16:39,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: writing mira0435601-192 from 0 to 778240 tid 1 2014-07-01 16:16:39,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: writing mira0435601-192 from 778240 to 1179648 tid 2 2014-07-01 16:16:39,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: write initial oid 193 2014-07-01 16:16:39,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: seq_num 192 ranges {0=647168,647168=401408} 2014-07-01 16:16:39,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: writing mira0435601-193 from 0 to 647168 tid 1 2014-07-01 16:16:39,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: writing mira0435601-193 from 647168 to 1048576 tid 2 2014-07-01 16:16:39,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:40,269.269 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 34 16 216 200 23.5242 32 1.6115 2.61202 2014-07-01 16:16:41,270.270 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 35 16 216 200 22.852 0 - 2.61202 2014-07-01 16:16:42,269.269 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 36 16 218 202 22.4394 4 0.61718 2.59853 2014-07-01 16:16:42,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 1 to mira0435601-183 2014-07-01 16:16:42,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 2 to mira0435601-183 2014-07-01 16:16:42,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 3 to mira0435601-183 2014-07-01 16:16:42,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 5 to mira0435601-183 2014-07-01 16:16:42,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 1 to mira0435601-182 2014-07-01 16:16:42,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 2 to mira0435601-182 2014-07-01 16:16:42,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 3 to mira0435601-182 2014-07-01 16:16:42,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 5 to mira0435601-182 2014-07-01 16:16:42,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 6 to mira0435601-182 2014-07-01 16:16:42,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 26 (ObjNum 181 snap 0 seq_num 181) dirty exists 2014-07-01 16:16:42,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: done (15 left) 2014-07-01 16:16:42,681.681 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:42,682.682 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:42,682.682 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:16:42,682.682 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: write initial oid 194 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: seq_num 193 ranges {0=675840,675840=610304,1286144=438272,1724416=557056} 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0435601-194 from 0 to 675840 tid 1 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0435601-194 from 675840 to 1286144 tid 2 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0435601-194 from 1286144 to 1724416 tid 3 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0435601-194 from 1724416 to 2281472 tid 4 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 6 to mira0435601-183 2014-07-01 16:16:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 46 (ObjNum 182 snap 0 seq_num 182) dirty exists 2014-07-01 16:16:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: done (15 left) 2014-07-01 16:16:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: write initial oid 195 2014-07-01 16:16:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: seq_num 194 ranges {0=598016} 2014-07-01 16:16:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: writing mira0435601-195 from 0 to 598016 tid 1 2014-07-01 16:16:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:43,270.270 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 37 16 220 204 22.0491 8 0.067553 2.59583 2014-07-01 16:16:43,422.422 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:43,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 1 to mira0435601-185 2014-07-01 16:16:43,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 3 to mira0435601-185 2014-07-01 16:16:43,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 4 to mira0435601-185 2014-07-01 16:16:43,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 52 (ObjNum 184 snap 0 seq_num 184) dirty exists 2014-07-01 16:16:43,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: done (15 left) 2014-07-01 16:16:43,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: write initial oid 196 2014-07-01 16:16:43,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: seq_num 195 ranges {0=622592,622592=778240,1400832=425984} 2014-07-01 16:16:43,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0435601-196 from 0 to 622592 tid 1 2014-07-01 16:16:43,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0435601-196 from 622592 to 1400832 tid 2 2014-07-01 16:16:43,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0435601-196 from 1400832 to 1826816 tid 3 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 1 to mira0435601-184 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 2 to mira0435601-184 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 3 to mira0435601-184 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 5 to mira0435601-184 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 6 to mira0435601-184 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 31 (ObjNum 183 snap 0 seq_num 183) dirty exists 2014-07-01 16:16:43,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: done (15 left) 2014-07-01 16:16:43,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: write initial oid 197 2014-07-01 16:16:43,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: seq_num 196 ranges {0=495616,495616=606208,1101824=598016} 2014-07-01 16:16:43,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0435601-197 from 0 to 495616 tid 1 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0435601-197 from 495616 to 1101824 tid 2 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0435601-197 from 1101824 to 1699840 tid 3 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 1 to mira0435601-193 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 2 to mira0435601-193 2014-07-01 16:16:43,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 4 to mira0435601-193 2014-07-01 16:16:43,966.966 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:16:43,966.966 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 26' 2014-07-01 16:16:44,269.269 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 38 16 222 206 21.6793 8 4.06128 2.60981 2014-07-01 16:16:44,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 1 to mira0435601-194 2014-07-01 16:16:44,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 1 to mira0435601-195 2014-07-01 16:16:44,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 2 to mira0435601-194 2014-07-01 16:16:44,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 3 to mira0435601-194 2014-07-01 16:16:44,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 3 to mira0435601-195 2014-07-01 16:16:44,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 4 to mira0435601-194 2014-07-01 16:16:44,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 6 to mira0435601-194 2014-07-01 16:16:45,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 1 to mira0435601-186 2014-07-01 16:16:45,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 3 to mira0435601-186 2014-07-01 16:16:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 1 to mira0435601-188 2014-07-01 16:16:45,094.094 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 26 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 2 to mira0435601-188 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 3 to mira0435601-188 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 4 to mira0435601-188 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 6 to mira0435601-188 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 1 to mira0435601-189 2014-07-01 16:16:45,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 1 to mira0435601-196 2014-07-01 16:16:45,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 2 to mira0435601-196 2014-07-01 16:16:45,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 3 to mira0435601-196 2014-07-01 16:16:45,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 5 to mira0435601-196 2014-07-01 16:16:45,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 1 to mira0435601-192 2014-07-01 16:16:45,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 2 to mira0435601-192 2014-07-01 16:16:45,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 4 to mira0435601-192 2014-07-01 16:16:45,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 2 to mira0435601-189 2014-07-01 16:16:45,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 3 to mira0435601-189 2014-07-01 16:16:45,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 5 to mira0435601-189 2014-07-01 16:16:45,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 1 to mira0435601-191 2014-07-01 16:16:45,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 2 to mira0435601-191 2014-07-01 16:16:45,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 3 to mira0435601-191 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 5 to mira0435601-191 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 4 to mira0435601-186 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 17 (ObjNum 185 snap 0 seq_num 185) dirty exists 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: done (15 left) 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: write initial oid 198 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: seq_num 197 ranges {0=770048,770048=589824,1359872=212992} 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: writing mira0435601-198 from 0 to 770048 tid 1 2014-07-01 16:16:45,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: writing mira0435601-198 from 770048 to 1359872 tid 2 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: writing mira0435601-198 from 1359872 to 1572864 tid 3 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 5 to mira0435601-192 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 52 (ObjNum 191 snap 0 seq_num 191) dirty exists 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: done (15 left) 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: write initial oid 199 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: seq_num 198 ranges {0=393216} 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: writing mira0435601-199 from 0 to 393216 tid 1 2014-07-01 16:16:45,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:45,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 39 16 233 217 22.2514 44 0.137892 2.75658 2014-07-01 16:16:45,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 1 to mira0435601-197 2014-07-01 16:16:45,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 2 to mira0435601-197 2014-07-01 16:16:45,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 3 to mira0435601-197 2014-07-01 16:16:45,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 5 to mira0435601-197 2014-07-01 16:16:45,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 6 to mira0435601-197 2014-07-01 16:16:45,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 197 v 56 (ObjNum 196 snap 0 seq_num 196) dirty exists 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: done (15 left) 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: write initial oid 200 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: seq_num 199 ranges {0=716800,716800=401408} 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: writing mira0435601-200 from 0 to 716800 tid 1 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: writing mira0435601-200 from 716800 to 1118208 tid 2 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:46,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 1 to mira0435601-198 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 2 to mira0435601-198 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 3 to mira0435601-198 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 5 to mira0435601-198 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 5 to mira0435601-193 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 46 (ObjNum 192 snap 0 seq_num 192) dirty exists 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: done (15 left) 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: write initial oid 201 2014-07-01 16:16:46,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: seq_num 200 ranges {0=589824,589824=401408} 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: writing mira0435601-201 from 0 to 589824 tid 1 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: writing mira0435601-201 from 589824 to 991232 tid 2 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 7 to mira0435601-188 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 22 (ObjNum 187 snap 0 seq_num 187) dirty exists 2014-07-01 16:16:46,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 6 to mira0435601-191 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 58 (ObjNum 190 snap 0 seq_num 190) dirty exists 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 6 to mira0435601-189 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 54 (ObjNum 188 snap 0 seq_num 188) dirty exists 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 7 to mira0435601-194 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 51 (ObjNum 193 snap 0 seq_num 193) dirty exists 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 4 to mira0435601-195 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 39 (ObjNum 194 snap 0 seq_num 194) dirty exists 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: done (15 left) 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: done (14 left) 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: done (13 left) 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: done (12 left) 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: done (11 left) 2014-07-01 16:16:46,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: write initial oid 202 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: seq_num 201 ranges {0=614400,614400=417792,1032192=733184,1765376=454656} 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0435601-202 from 0 to 614400 tid 1 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0435601-202 from 614400 to 1032192 tid 2 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0435601-202 from 1032192 to 1765376 tid 3 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0435601-202 from 1765376 to 2220032 tid 4 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: write initial oid 203 2014-07-01 16:16:46,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: seq_num 202 ranges {0=487424,487424=598016,1085440=557056,1642496=450560} 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0435601-203 from 0 to 487424 tid 1 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0435601-203 from 487424 to 1085440 tid 2 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0435601-203 from 1085440 to 1642496 tid 3 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0435601-203 from 1642496 to 2093056 tid 4 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: write initial oid 204 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: seq_num 203 ranges {0=409600} 2014-07-01 16:16:46,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: writing mira0435601-204 from 0 to 409600 tid 1 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: write initial oid 205 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: seq_num 204 ranges {0=434176,434176=413696,847872=503808,1351680=286720} 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0435601-205 from 0 to 434176 tid 1 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0435601-205 from 434176 to 847872 tid 2 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0435601-205 from 847872 to 1351680 tid 3 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0435601-205 from 1351680 to 1638400 tid 4 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: write initial oid 206 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: seq_num 205 ranges {0=462848} 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: writing mira0435601-206 from 0 to 462848 tid 1 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 6 to mira0435601-196 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 55 (ObjNum 195 snap 0 seq_num 195) dirty exists 2014-07-01 16:16:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: done (15 left) 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: write initial oid 207 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: seq_num 206 ranges {0=581632,581632=577536,1159168=225280} 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: writing mira0435601-207 from 0 to 581632 tid 1 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: writing mira0435601-207 from 581632 to 1159168 tid 2 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: writing mira0435601-207 from 1159168 to 1384448 tid 3 2014-07-01 16:16:46,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:46,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:16:46.183002min lat: 0.067553 max lat: 9.21429 avg lat: 2.741 2014-07-01 16:16:46,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:16:46,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 40 16 235 219 21.8951 8 1.12822 2.741 2014-07-01 16:16:47,270.270 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 41 16 236 220 21.4586 4 1.46094 2.73518 2014-07-01 16:16:47,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 1 to mira0435601-200 2014-07-01 16:16:47,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 1 to mira0435601-170 2014-07-01 16:16:47,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 2 to mira0435601-170 2014-07-01 16:16:47,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 1 to mira0435601-199 2014-07-01 16:16:47,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 3 to mira0435601-199 2014-07-01 16:16:47,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 3 to mira0435601-170 2014-07-01 16:16:47,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 4 to mira0435601-170 2014-07-01 16:16:47,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 6 to mira0435601-170 2014-07-01 16:16:47,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 1 to mira0435601-187 2014-07-01 16:16:47,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 3 to mira0435601-187 2014-07-01 16:16:47,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 1 to mira0435601-190 2014-07-01 16:16:47,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 2 to mira0435601-190 2014-07-01 16:16:47,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 2 to mira0435601-200 2014-07-01 16:16:47,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 4 to mira0435601-200 2014-07-01 16:16:47,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 3 to mira0435601-190 2014-07-01 16:16:47,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 5 to mira0435601-190 2014-07-01 16:16:47,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 1 to mira0435601-202 2014-07-01 16:16:47,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 2 to mira0435601-202 2014-07-01 16:16:47,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 3 to mira0435601-202 2014-07-01 16:16:47,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 4 to mira0435601-202 2014-07-01 16:16:47,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 6 to mira0435601-202 2014-07-01 16:16:47,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 1 to mira0435601-203 2014-07-01 16:16:47,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 2 to mira0435601-203 2014-07-01 16:16:47,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 3 to mira0435601-203 2014-07-01 16:16:47,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 4 to mira0435601-203 2014-07-01 16:16:47,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 6 to mira0435601-203 2014-07-01 16:16:48,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 1 to mira0435601-201 2014-07-01 16:16:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 2 to mira0435601-201 2014-07-01 16:16:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 4 to mira0435601-201 2014-07-01 16:16:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 1 to mira0435601-204 2014-07-01 16:16:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 3 to mira0435601-204 2014-07-01 16:16:48,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 1 to mira0435601-205 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 2 to mira0435601-205 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 3 to mira0435601-205 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 6 to mira0435601-198 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 54 (ObjNum 197 snap 0 seq_num 197) dirty exists 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: done (15 left) 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: write initial oid 208 2014-07-01 16:16:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: seq_num 207 ranges {0=204800} 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: writing mira0435601-208 from 0 to 204800 tid 1 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 4 to mira0435601-199 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 39 (ObjNum 198 snap 0 seq_num 198) dirty exists 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: done (15 left) 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: write initial oid 209 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: seq_num 208 ranges {0=528384,528384=401408} 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: writing mira0435601-209 from 0 to 528384 tid 1 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: writing mira0435601-209 from 528384 to 929792 tid 2 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 1 to mira0435601-155 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 1 to mira0435601-207 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 2 to mira0435601-207 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 3 to mira0435601-207 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 5 to mira0435601-207 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 3 to mira0435601-155 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 4 to mira0435601-205 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 6 to mira0435601-205 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 7 to mira0435601-170 2014-07-01 16:16:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 56 (ObjNum 169 snap 0 seq_num 169) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: done (15 left) 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: write initial oid 210 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: seq_num 209 ranges {0=802816} 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: writing mira0435601-210 from 0 to 802816 tid 1 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 4 to mira0435601-187 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 58 (ObjNum 186 snap 0 seq_num 186) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 5 to mira0435601-200 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 37 (ObjNum 199 snap 0 seq_num 199) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 4 to mira0435601-155 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 42 (ObjNum 154 snap 0 seq_num 154) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 6 to mira0435601-190 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 62 (ObjNum 189 snap 0 seq_num 189) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 5 to mira0435601-201 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 201 v 61 (ObjNum 200 snap 0 seq_num 200) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 1 to mira0435601-209 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 7 to mira0435601-203 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 60 (ObjNum 202 snap 0 seq_num 202) dirty exists 2014-07-01 16:16:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 4 to mira0435601-204 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 63 (ObjNum 203 snap 0 seq_num 203) dirty exists 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 2 to mira0435601-209 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 4 to mira0435601-209 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: done (15 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: done (14 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: done (13 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: done (12 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: done (11 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: done (10 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: done (9 left) 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: write initial oid 211 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: seq_num 210 ranges {0=425984,425984=802816,1228800=499712,1728512=303104} 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0435601-211 from 0 to 425984 tid 1 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0435601-211 from 425984 to 1228800 tid 2 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0435601-211 from 1228800 to 1728512 tid 3 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0435601-211 from 1728512 to 2031616 tid 4 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: write initial oid 212 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: seq_num 211 ranges {0=700416,700416=790528,1490944=413696} 2014-07-01 16:16:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0435601-212 from 0 to 700416 tid 1 2014-07-01 16:16:48,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0435601-212 from 700416 to 1490944 tid 2 2014-07-01 16:16:48,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 42 16 241 225 21.4238 20 0.470782 2.71366 2014-07-01 16:16:48,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0435601-212 from 1490944 to 1904640 tid 3 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: write initial oid 213 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: seq_num 212 ranges {0=217088} 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: writing mira0435601-213 from 0 to 217088 tid 1 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: write initial oid 214 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: seq_num 213 ranges {0=647168,647168=602112,1249280=200704} 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: writing mira0435601-214 from 0 to 647168 tid 1 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: writing mira0435601-214 from 647168 to 1249280 tid 2 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: writing mira0435601-214 from 1249280 to 1449984 tid 3 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: write initial oid 215 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: seq_num 214 ranges {0=520192,520192=786432,1306624=16384} 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: writing mira0435601-215 from 0 to 520192 tid 1 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: writing mira0435601-215 from 520192 to 1306624 tid 2 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: writing mira0435601-215 from 1306624 to 1323008 tid 3 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: write initial oid 216 2014-07-01 16:16:48,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: seq_num 215 ranges {0=790528,790528=401408} 2014-07-01 16:16:48,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: writing mira0435601-216 from 0 to 790528 tid 1 2014-07-01 16:16:48,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: writing mira0435601-216 from 790528 to 1191936 tid 2 2014-07-01 16:16:48,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: write initial oid 217 2014-07-01 16:16:48,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: seq_num 216 ranges {0=16384} 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: writing mira0435601-217 from 0 to 16384 tid 1 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 7 to mira0435601-205 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 205 v 27 (ObjNum 204 snap 0 seq_num 204) dirty exists 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 1 to mira0435601-208 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 3 to mira0435601-208 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 5 to mira0435601-209 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 209 v 43 (ObjNum 208 snap 0 seq_num 208) dirty exists 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 4 to mira0435601-208 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 49 (ObjNum 207 snap 0 seq_num 207) dirty exists 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 1 to mira0435601-210 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 3 to mira0435601-210 2014-07-01 16:16:48,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 7 to mira0435601-202 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 30 (ObjNum 201 snap 0 seq_num 201) dirty exists 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 4 to mira0435601-210 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 210 v 51 (ObjNum 209 snap 0 seq_num 209) dirty exists 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 1 to mira0435601-206 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 3 to mira0435601-206 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 4 to mira0435601-206 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 41 (ObjNum 205 snap 0 seq_num 205) dirty exists 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: done (15 left) 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: done (14 left) 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: done (13 left) 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: done (12 left) 2014-07-01 16:16:48,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: done (11 left) 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: done (10 left) 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: write initial oid 218 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: seq_num 217 ranges {0=741376} 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: writing mira0435601-218 from 0 to 741376 tid 1 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: write initial oid 219 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: seq_num 218 ranges {0=765952,765952=765952,1531904=438272} 2014-07-01 16:16:48,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: writing mira0435601-219 from 0 to 765952 tid 1 2014-07-01 16:16:48,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: writing mira0435601-219 from 765952 to 1531904 tid 2 2014-07-01 16:16:48,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: writing mira0435601-219 from 1531904 to 1970176 tid 3 2014-07-01 16:16:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: write initial oid 220 2014-07-01 16:16:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: seq_num 219 ranges {0=638976,638976=749568,1388544=454656} 2014-07-01 16:16:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0435601-220 from 0 to 638976 tid 1 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0435601-220 from 638976 to 1388544 tid 2 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0435601-220 from 1388544 to 1843200 tid 3 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: write initial oid 221 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: seq_num 220 ranges {0=512000,512000=581632,1093632=622592} 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0435601-221 from 0 to 512000 tid 1 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0435601-221 from 512000 to 1093632 tid 2 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0435601-221 from 1093632 to 1716224 tid 3 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: write initial oid 222 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: seq_num 221 ranges {0=585728,585728=409600,995328=393216} 2014-07-01 16:16:48,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0435601-222 from 0 to 585728 tid 1 2014-07-01 16:16:48,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0435601-222 from 585728 to 995328 tid 2 2014-07-01 16:16:48,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0435601-222 from 995328 to 1388544 tid 3 2014-07-01 16:16:48,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: write initial oid 223 2014-07-01 16:16:48,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: seq_num 222 ranges {0=458752,458752=638976,1097728=163840} 2014-07-01 16:16:48,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: writing mira0435601-223 from 0 to 458752 tid 1 2014-07-01 16:16:48,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: writing mira0435601-223 from 458752 to 1097728 tid 2 2014-07-01 16:16:48,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: writing mira0435601-223 from 1097728 to 1261568 tid 3 2014-07-01 16:16:48,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 1 to mira0435601-211 2014-07-01 16:16:48,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 2 to mira0435601-211 2014-07-01 16:16:48,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 3 to mira0435601-211 2014-07-01 16:16:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 4 to mira0435601-211 2014-07-01 16:16:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 6 to mira0435601-211 2014-07-01 16:16:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 7 to mira0435601-211 2014-07-01 16:16:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 57 (ObjNum 210 snap 0 seq_num 210) dirty exists 2014-07-01 16:16:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: done (15 left) 2014-07-01 16:16:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: write initial oid 224 2014-07-01 16:16:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: seq_num 223 ranges {0=733184,733184=401408} 2014-07-01 16:16:48,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: writing mira0435601-224 from 0 to 733184 tid 1 2014-07-01 16:16:48,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: writing mira0435601-224 from 733184 to 1134592 tid 2 2014-07-01 16:16:48,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 6 to mira0435601-207 2014-07-01 16:16:48,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 207 v 66 (ObjNum 206 snap 0 seq_num 206) dirty exists 2014-07-01 16:16:48,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: done (15 left) 2014-07-01 16:16:48,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: write initial oid 225 2014-07-01 16:16:48,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: seq_num 224 ranges {0=757760,757760=405504,1163264=585728,1748992=495616,2244608=118784} 2014-07-01 16:16:48,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0435601-225 from 0 to 757760 tid 1 2014-07-01 16:16:48,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0435601-225 from 757760 to 1163264 tid 2 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0435601-225 from 1163264 to 1748992 tid 3 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0435601-225 from 1748992 to 2244608 tid 4 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0435601-225 from 2244608 to 2363392 tid 5 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 1 to mira0435601-212 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 2 to mira0435601-212 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 3 to mira0435601-212 2014-07-01 16:16:48,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 5 to mira0435601-212 2014-07-01 16:16:48,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 1 to mira0435601-213 2014-07-01 16:16:48,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 3 to mira0435601-213 2014-07-01 16:16:49,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 1 to mira0435601-214 2014-07-01 16:16:49,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 2 to mira0435601-214 2014-07-01 16:16:49,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 3 to mira0435601-214 2014-07-01 16:16:49,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 5 to mira0435601-214 2014-07-01 16:16:49,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 1 to mira0435601-217 2014-07-01 16:16:49,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 3 to mira0435601-217 2014-07-01 16:16:49,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 6 to mira0435601-212 2014-07-01 16:16:49,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 50 (ObjNum 211 snap 0 seq_num 211) dirty exists 2014-07-01 16:16:49,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: done (15 left) 2014-07-01 16:16:49,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: write initial oid 226 2014-07-01 16:16:49,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: seq_num 225 ranges {0=630784,630784=790528,1421312=458752,1880064=356352} 2014-07-01 16:16:49,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0435601-226 from 0 to 630784 tid 1 2014-07-01 16:16:49,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0435601-226 from 630784 to 1421312 tid 2 2014-07-01 16:16:49,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0435601-226 from 1421312 to 1880064 tid 3 2014-07-01 16:16:49,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0435601-226 from 1880064 to 2236416 tid 4 2014-07-01 16:16:49,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 4 to mira0435601-213 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 52 (ObjNum 212 snap 0 seq_num 212) dirty exists 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: done (15 left) 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: write initial oid 227 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: seq_num 226 ranges {0=552960} 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: writing mira0435601-227 from 0 to 552960 tid 1 2014-07-01 16:16:49,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:49,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 43 16 258 242 22.5067 68 0.938069 2.72499 2014-07-01 16:16:49,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 6 to mira0435601-214 2014-07-01 16:16:49,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 56 (ObjNum 213 snap 0 seq_num 213) dirty exists 2014-07-01 16:16:49,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: done (15 left) 2014-07-01 16:16:49,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: write initial oid 228 2014-07-01 16:16:49,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: seq_num 227 ranges {0=577536,577536=557056,1134592=647168} 2014-07-01 16:16:49,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0435601-228 from 0 to 577536 tid 1 2014-07-01 16:16:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0435601-228 from 577536 to 1134592 tid 2 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0435601-228 from 1134592 to 1781760 tid 3 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 1 to mira0435601-216 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 2 to mira0435601-216 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 4 to mira0435601-216 2014-07-01 16:16:49,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 5 to mira0435601-216 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 216 v 29 (ObjNum 215 snap 0 seq_num 215) dirty exists 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 4 to mira0435601-217 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 58 (ObjNum 216 snap 0 seq_num 216) dirty exists 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: done (15 left) 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: done (14 left) 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: write initial oid 229 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: seq_num 228 ranges {0=450560,450560=786432,1236992=417792} 2014-07-01 16:16:49,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0435601-229 from 0 to 450560 tid 1 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0435601-229 from 450560 to 1236992 tid 2 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0435601-229 from 1236992 to 1654784 tid 3 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: write initial oid 230 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: seq_num 229 ranges {0=724992,724992=774144,1499136=28672} 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: writing mira0435601-230 from 0 to 724992 tid 1 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: writing mira0435601-230 from 724992 to 1499136 tid 2 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: writing mira0435601-230 from 1499136 to 1527808 tid 3 2014-07-01 16:16:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:50,104.104 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:50,104.104 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:50,105.105 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 3 to 0.790933952363 2014-07-01 16:16:50,105.105 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 0.790933952363' 2014-07-01 16:16:50,270.270 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 44 16 265 249 22.6314 28 1.22096 2.71041 2014-07-01 16:16:50,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 1 to mira0435601-226 2014-07-01 16:16:50,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 2 to mira0435601-226 2014-07-01 16:16:50,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 3 to mira0435601-226 2014-07-01 16:16:50,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 4 to mira0435601-226 2014-07-01 16:16:50,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 6 to mira0435601-226 2014-07-01 16:16:50,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 1 to mira0435601-221 2014-07-01 16:16:50,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 1 to mira0435601-157 2014-07-01 16:16:50,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 2 to mira0435601-221 2014-07-01 16:16:50,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 2 to mira0435601-157 2014-07-01 16:16:50,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 3 to mira0435601-221 2014-07-01 16:16:50,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 3 to mira0435601-157 2014-07-01 16:16:50,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 4 to mira0435601-157 2014-07-01 16:16:50,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 5 to mira0435601-221 2014-07-01 16:16:50,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 6 to mira0435601-157 2014-07-01 16:16:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 7 to mira0435601-157 2014-07-01 16:16:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 47 (ObjNum 156 snap 0 seq_num 156) dirty exists 2014-07-01 16:16:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: done (15 left) 2014-07-01 16:16:50,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: write initial oid 231 2014-07-01 16:16:50,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: seq_num 230 ranges {0=798720,798720=401408} 2014-07-01 16:16:50,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: writing mira0435601-231 from 0 to 798720 tid 1 2014-07-01 16:16:50,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: writing mira0435601-231 from 798720 to 1200128 tid 2 2014-07-01 16:16:50,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:50,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 6 to mira0435601-221 2014-07-01 16:16:50,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 55 (ObjNum 220 snap 0 seq_num 220) dirty exists 2014-07-01 16:16:50,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: done (15 left) 2014-07-01 16:16:50,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: write initial oid 232 2014-07-01 16:16:50,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: seq_num 231 ranges {0=24576} 2014-07-01 16:16:50,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: writing mira0435601-232 from 0 to 24576 tid 1 2014-07-01 16:16:50,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:51,981.981 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 45 16 267 251 22.3062 8 5.35037 2.71849 2014-07-01 16:16:51,982.982 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 0.790934 (8518342) 2014-07-01 16:16:51,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 1 to mira0435601-225 2014-07-01 16:16:51,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 2 to mira0435601-225 2014-07-01 16:16:51,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 3 to mira0435601-225 2014-07-01 16:16:51,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 4 to mira0435601-225 2014-07-01 16:16:51,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 5 to mira0435601-225 2014-07-01 16:16:51,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 7 to mira0435601-225 2014-07-01 16:16:51,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 1 to mira0435601-222 2014-07-01 16:16:51,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 2 to mira0435601-222 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 3 to mira0435601-222 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 5 to mira0435601-222 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 1 to mira0435601-223 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 2 to mira0435601-223 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 3 to mira0435601-223 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 5 to mira0435601-223 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 1 to mira0435601-166 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 2 to mira0435601-166 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 8 to mira0435601-225 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 64 (ObjNum 224 snap 0 seq_num 224) dirty exists 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: done (15 left) 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: write initial oid 233 2014-07-01 16:16:51,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: seq_num 232 ranges {0=544768,544768=401408} 2014-07-01 16:16:52,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: writing mira0435601-233 from 0 to 544768 tid 1 2014-07-01 16:16:52,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: writing mira0435601-233 from 544768 to 946176 tid 2 2014-07-01 16:16:52,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:52,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 1 to mira0435601-215 2014-07-01 16:16:52,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 2 to mira0435601-215 2014-07-01 16:16:52,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 3 to mira0435601-215 2014-07-01 16:16:52,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 5 to mira0435601-215 2014-07-01 16:16:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 1 to mira0435601-219 2014-07-01 16:16:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 2 to mira0435601-219 2014-07-01 16:16:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 3 to mira0435601-219 2014-07-01 16:16:52,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 5 to mira0435601-219 2014-07-01 16:16:52,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 1 to mira0435601-220 2014-07-01 16:16:52,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 1 to mira0435601-218 2014-07-01 16:16:52,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 2 to mira0435601-220 2014-07-01 16:16:52,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 3 to mira0435601-218 2014-07-01 16:16:52,270.270 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 46 16 274 258 22.4299 28 3.56813 2.71486 2014-07-01 16:16:52,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 3 to mira0435601-220 2014-07-01 16:16:52,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 5 to mira0435601-220 2014-07-01 16:16:52,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 1 to mira0435601-224 2014-07-01 16:16:52,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 2 to mira0435601-224 2014-07-01 16:16:52,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 4 to mira0435601-224 2014-07-01 16:16:53,272.272 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 47 16 287 271 23.0588 52 0.138894 2.66796 2014-07-01 16:16:53,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 6 to mira0435601-215 2014-07-01 16:16:53,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 35 (ObjNum 214 snap 0 seq_num 214) dirty exists 2014-07-01 16:16:53,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: done (15 left) 2014-07-01 16:16:53,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: write initial oid 234 2014-07-01 16:16:53,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: seq_num 233 ranges {0=569344,569344=598016,1167360=507904,1675264=499712} 2014-07-01 16:16:53,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0435601-234 from 0 to 569344 tid 1 2014-07-01 16:16:53,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0435601-234 from 569344 to 1167360 tid 2 2014-07-01 16:16:53,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0435601-234 from 1167360 to 1675264 tid 3 2014-07-01 16:16:54,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0435601-234 from 1675264 to 2174976 tid 4 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 4 to mira0435601-218 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 65 (ObjNum 217 snap 0 seq_num 217) dirty exists 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 6 to mira0435601-219 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 64 (ObjNum 218 snap 0 seq_num 218) dirty exists 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 6 to mira0435601-220 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 39 (ObjNum 219 snap 0 seq_num 219) dirty exists 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 5 to mira0435601-224 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 42 (ObjNum 223 snap 0 seq_num 223) dirty exists 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: done (15 left) 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: done (14 left) 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: done (13 left) 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: done (12 left) 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: write initial oid 235 2014-07-01 16:16:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: seq_num 234 ranges {0=598016,598016=401408} 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: writing mira0435601-235 from 0 to 598016 tid 1 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: writing mira0435601-235 from 598016 to 999424 tid 2 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: write initial oid 236 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: seq_num 235 ranges {0=360448} 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: writing mira0435601-236 from 0 to 360448 tid 1 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: write initial oid 237 2014-07-01 16:16:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: seq_num 236 ranges {0=790528,790528=593920,1384448=208896} 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0435601-237 from 0 to 790528 tid 1 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0435601-237 from 790528 to 1384448 tid 2 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0435601-237 from 1384448 to 1593344 tid 3 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: write initial oid 238 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: seq_num 237 ranges {0=417792} 2014-07-01 16:16:54,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: writing mira0435601-238 from 0 to 417792 tid 1 2014-07-01 16:16:54,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:54,271.271 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 48 16 289 273 22.745 8 1.15164 2.65685 2014-07-01 16:16:55,272.272 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 49 16 291 275 22.4441 8 1.2247 2.64691 2014-07-01 16:16:56,272.272 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 50 16 291 275 21.9952 0 - 2.64691 2014-07-01 16:16:56,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 6 to mira0435601-222 2014-07-01 16:16:56,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 70 (ObjNum 221 snap 0 seq_num 221) dirty exists 2014-07-01 16:16:56,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: done (15 left) 2014-07-01 16:16:56,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: write initial oid 239 2014-07-01 16:16:56,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: seq_num 238 ranges {0=532480,532480=405504,937984=397312} 2014-07-01 16:16:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: writing mira0435601-239 from 0 to 532480 tid 1 2014-07-01 16:16:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: writing mira0435601-239 from 532480 to 937984 tid 2 2014-07-01 16:16:56,982.982 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:16:56,982.982 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:16:56,982.982 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 16:16:56,983.983 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: writing mira0435601-239 from 937984 to 1335296 tid 3 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 6 to mira0435601-223 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 74 (ObjNum 222 snap 0 seq_num 222) dirty exists 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 7 to mira0435601-226 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 42 (ObjNum 225 snap 0 seq_num 225) dirty exists 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: done (15 left) 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: done (14 left) 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: write initial oid 240 2014-07-01 16:16:57,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: seq_num 239 ranges {0=610304,610304=401408} 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: writing mira0435601-240 from 0 to 610304 tid 1 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: writing mira0435601-240 from 610304 to 1011712 tid 2 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: write initial oid 241 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: seq_num 240 ranges {0=638976,638976=573440,1212416=573440,1785856=458752} 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: writing mira0435601-241 from 0 to 638976 tid 1 2014-07-01 16:16:57,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: writing mira0435601-241 from 638976 to 1212416 tid 2 2014-07-01 16:16:57,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: writing mira0435601-241 from 1212416 to 1785856 tid 3 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: writing mira0435601-241 from 1785856 to 2244608 tid 4 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 1 to mira0435601-228 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 2 to mira0435601-228 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 3 to mira0435601-228 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 5 to mira0435601-228 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 1 to mira0435601-229 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 1 to mira0435601-227 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 1 to mira0435601-230 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 3 to mira0435601-227 2014-07-01 16:16:57,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 2 to mira0435601-230 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 4 to mira0435601-227 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 76 (ObjNum 226 snap 0 seq_num 226) dirty exists 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: done (15 left) 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: write initial oid 242 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: seq_num 241 ranges {0=753664} 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: writing mira0435601-242 from 0 to 753664 tid 1 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 1 to mira0435601-238 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 3 to mira0435601-238 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 3 to mira0435601-230 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 2 to mira0435601-229 2014-07-01 16:16:57,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 5 to mira0435601-230 2014-07-01 16:16:57,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 6 to mira0435601-228 2014-07-01 16:16:57,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 47 (ObjNum 227 snap 0 seq_num 227) dirty exists 2014-07-01 16:16:57,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: done (15 left) 2014-07-01 16:16:57,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: write initial oid 243 2014-07-01 16:16:57,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: seq_num 242 ranges {0=782336,782336=790528,1572864=413696} 2014-07-01 16:16:57,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0435601-243 from 0 to 782336 tid 1 2014-07-01 16:16:57,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0435601-243 from 782336 to 1572864 tid 2 2014-07-01 16:16:57,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0435601-243 from 1572864 to 1986560 tid 3 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 3 to mira0435601-229 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 5 to mira0435601-229 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 1 to mira0435601-231 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 2 to mira0435601-231 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 6 to mira0435601-230 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 45 (ObjNum 229 snap 0 seq_num 229) dirty exists 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 4 to mira0435601-231 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 1 to mira0435601-232 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 3 to mira0435601-232 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 3 to mira0435601-166 2014-07-01 16:16:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 5 to mira0435601-166 2014-07-01 16:16:57,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 4 to mira0435601-232 2014-07-01 16:16:57,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 41 (ObjNum 231 snap 0 seq_num 231) dirty exists 2014-07-01 16:16:57,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 6 to mira0435601-229 2014-07-01 16:16:57,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 51 (ObjNum 228 snap 0 seq_num 228) dirty exists 2014-07-01 16:16:57,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 5 to mira0435601-231 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 60 (ObjNum 230 snap 0 seq_num 230) dirty exists 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 6 to mira0435601-166 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 78 (ObjNum 165 snap 0 seq_num 165) dirty exists 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 1 to mira0435601-233 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 2 to mira0435601-233 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: done (15 left) 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: done (14 left) 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: done (13 left) 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: done (12 left) 2014-07-01 16:16:57,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: done (11 left) 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: write initial oid 244 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: seq_num 243 ranges {0=409600,409600=401408} 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: writing mira0435601-244 from 0 to 409600 tid 1 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: writing mira0435601-244 from 409600 to 811008 tid 2 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: write initial oid 245 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: seq_num 244 ranges {0=729088,729088=798720,1527808=4096} 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0435601-245 from 0 to 729088 tid 1 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0435601-245 from 729088 to 1527808 tid 2 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0435601-245 from 1527808 to 1531904 tid 3 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: write initial oid 246 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: seq_num 245 ranges {0=602112,602112=786432,1388544=16384} 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: writing mira0435601-246 from 0 to 602112 tid 1 2014-07-01 16:16:57,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: writing mira0435601-246 from 602112 to 1388544 tid 2 2014-07-01 16:16:57,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: writing mira0435601-246 from 1388544 to 1404928 tid 3 2014-07-01 16:16:57,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: write initial oid 247 2014-07-01 16:16:57,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: seq_num 246 ranges {0=229376} 2014-07-01 16:16:57,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: writing mira0435601-247 from 0 to 229376 tid 1 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: write initial oid 248 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: seq_num 247 ranges {0=98304} 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: writing mira0435601-248 from 0 to 98304 tid 1 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 4 to mira0435601-233 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 5 to mira0435601-233 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 54 (ObjNum 232 snap 0 seq_num 232) dirty exists 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: done (15 left) 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: write initial oid 249 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: seq_num 248 ranges {0=421888,421888=401408} 2014-07-01 16:16:57,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: writing mira0435601-249 from 0 to 421888 tid 1 2014-07-01 16:16:57,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: writing mira0435601-249 from 421888 to 823296 tid 2 2014-07-01 16:16:57,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 51 16 306 290 22.7401 30 0.200412 2.7407 2014-07-01 16:16:57,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 1 to mira0435601-235 2014-07-01 16:16:57,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 2 to mira0435601-235 2014-07-01 16:16:57,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 1 to mira0435601-234 2014-07-01 16:16:57,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 2 to mira0435601-234 2014-07-01 16:16:57,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 3 to mira0435601-234 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 4 to mira0435601-234 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 4 to mira0435601-238 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 56 (ObjNum 237 snap 0 seq_num 237) dirty exists 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: done (15 left) 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: write initial oid 250 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: seq_num 249 ranges {0=450560,450560=765952,1216512=536576,1753088=303104} 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0435601-250 from 0 to 450560 tid 1 2014-07-01 16:16:57,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0435601-250 from 450560 to 1216512 tid 2 2014-07-01 16:16:57,586.586 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:16:57,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0435601-250 from 1216512 to 1753088 tid 3 2014-07-01 16:16:57,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0435601-250 from 1753088 to 2056192 tid 4 2014-07-01 16:16:57,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 1 to mira0435601-239 2014-07-01 16:16:57,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 2 to mira0435601-239 2014-07-01 16:16:57,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 1 to mira0435601-248 2014-07-01 16:16:57,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 3 to mira0435601-248 2014-07-01 16:16:57,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 4 to mira0435601-248 2014-07-01 16:16:57,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 44 (ObjNum 247 snap 0 seq_num 247) dirty exists 2014-07-01 16:16:57,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: done (15 left) 2014-07-01 16:16:57,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: write initial oid 251 2014-07-01 16:16:57,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: seq_num 250 ranges {0=720896,720896=593920,1314816=413696,1728512=196608} 2014-07-01 16:16:57,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0435601-251 from 0 to 720896 tid 1 2014-07-01 16:16:57,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0435601-251 from 720896 to 1314816 tid 2 2014-07-01 16:16:57,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0435601-251 from 1314816 to 1728512 tid 3 2014-07-01 16:16:57,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0435601-251 from 1728512 to 1925120 tid 4 2014-07-01 16:16:57,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:57,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 1 to mira0435601-250 2014-07-01 16:16:57,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 4 to mira0435601-235 2014-07-01 16:16:57,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 3 to mira0435601-239 2014-07-01 16:16:57,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 1 to mira0435601-241 2014-07-01 16:16:57,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 6 to mira0435601-234 2014-07-01 16:16:57,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 2 to mira0435601-241 2014-07-01 16:16:58,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 5 to mira0435601-239 2014-07-01 16:16:58,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 1 to mira0435601-240 2014-07-01 16:16:58,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 1 to mira0435601-236 2014-07-01 16:16:58,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 2 to mira0435601-240 2014-07-01 16:16:58,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 4 to mira0435601-240 2014-07-01 16:16:58,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 3 to mira0435601-236 2014-07-01 16:16:58,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 1 to mira0435601-237 2014-07-01 16:16:58,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 2 to mira0435601-237 2014-07-01 16:16:58,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 3 to mira0435601-237 2014-07-01 16:16:58,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 5 to mira0435601-237 2014-07-01 16:16:58,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 3 to mira0435601-241 2014-07-01 16:16:58,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 6 to mira0435601-239 2014-07-01 16:16:58,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 45 (ObjNum 238 snap 0 seq_num 238) dirty exists 2014-07-01 16:16:58,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: done (15 left) 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: write initial oid 252 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: seq_num 251 ranges {0=593920,593920=532480,1126400=638976,1765376=32768} 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0435601-252 from 0 to 593920 tid 1 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0435601-252 from 593920 to 1126400 tid 2 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0435601-252 from 1126400 to 1765376 tid 3 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0435601-252 from 1765376 to 1798144 tid 4 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 5 to mira0435601-240 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 48 (ObjNum 239 snap 0 seq_num 239) dirty exists 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 4 to mira0435601-236 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 29 (ObjNum 235 snap 0 seq_num 235) dirty exists 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 7 to mira0435601-234 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 60 (ObjNum 233 snap 0 seq_num 233) dirty exists 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 5 to mira0435601-235 2014-07-01 16:16:58,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 68 (ObjNum 234 snap 0 seq_num 234) dirty exists 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 6 to mira0435601-237 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 33 (ObjNum 236 snap 0 seq_num 236) dirty exists 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: done (15 left) 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: done (14 left) 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: done (13 left) 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: done (12 left) 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: done (11 left) 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: write initial oid 253 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: seq_num 252 ranges {0=466944,466944=761856,1228800=442368} 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0435601-253 from 0 to 466944 tid 1 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0435601-253 from 466944 to 1228800 tid 2 2014-07-01 16:16:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0435601-253 from 1228800 to 1671168 tid 3 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: write initial oid 254 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: seq_num 253 ranges {0=540672,540672=745472,1286144=57344} 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0435601-254 from 0 to 540672 tid 1 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0435601-254 from 540672 to 1286144 tid 2 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0435601-254 from 1286144 to 1343488 tid 3 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: write initial oid 255 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: seq_num 254 ranges {0=413696,413696=421888,835584=380928} 2014-07-01 16:16:58,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: writing mira0435601-255 from 0 to 413696 tid 1 2014-07-01 16:16:58,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: writing mira0435601-255 from 413696 to 835584 tid 2 2014-07-01 16:16:58,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: writing mira0435601-255 from 835584 to 1216512 tid 3 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: write initial oid 256 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: seq_num 255 ranges {0=684032,684032=401408} 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: writing mira0435601-256 from 0 to 684032 tid 1 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: writing mira0435601-256 from 684032 to 1085440 tid 2 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: write initial oid 257 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: seq_num 256 ranges {0=712704,712704=585728,1298432=761856,2060288=258048} 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: writing mira0435601-257 from 0 to 712704 tid 1 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: writing mira0435601-257 from 712704 to 1298432 tid 2 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: writing mira0435601-257 from 1298432 to 2060288 tid 3 2014-07-01 16:16:58,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: writing mira0435601-257 from 2060288 to 2318336 tid 4 2014-07-01 16:16:58,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:58,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 52 16 310 294 22.6104 16 0.961449 2.73144 2014-07-01 16:16:58,444.444 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:16:58,444.444 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 26' 2014-07-01 16:16:58,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 2 to mira0435601-250 2014-07-01 16:16:58,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 3 to mira0435601-250 2014-07-01 16:16:58,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 4 to mira0435601-250 2014-07-01 16:16:58,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 6 to mira0435601-250 2014-07-01 16:16:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 7 to mira0435601-250 2014-07-01 16:16:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 49 (ObjNum 249 snap 0 seq_num 249) dirty exists 2014-07-01 16:16:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: done (15 left) 2014-07-01 16:16:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: write initial oid 258 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: seq_num 257 ranges {0=790528,790528=573440,1363968=585728,1949696=45056} 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0435601-258 from 0 to 790528 tid 1 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0435601-258 from 790528 to 1363968 tid 2 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0435601-258 from 1363968 to 1949696 tid 3 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0435601-258 from 1949696 to 1994752 tid 4 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 1 to mira0435601-244 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 2 to mira0435601-244 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 4 to mira0435601-244 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 4 to mira0435601-241 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 6 to mira0435601-241 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 5 to mira0435601-244 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 59 (ObjNum 243 snap 0 seq_num 243) dirty exists 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: done (15 left) 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: write initial oid 259 2014-07-01 16:16:59,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: seq_num 258 ranges {0=503808} 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: writing mira0435601-259 from 0 to 503808 tid 1 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 1 to mira0435601-247 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 7 to mira0435601-241 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 73 (ObjNum 240 snap 0 seq_num 240) dirty exists 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 3 to mira0435601-247 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 1 to mira0435601-249 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: done (15 left) 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: write initial oid 260 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: seq_num 259 ranges {0=532480,532480=737280,1269760=466944} 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0435601-260 from 0 to 532480 tid 1 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0435601-260 from 532480 to 1269760 tid 2 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0435601-260 from 1269760 to 1736704 tid 3 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 2 to mira0435601-249 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 1 to mira0435601-251 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 2 to mira0435601-251 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 3 to mira0435601-251 2014-07-01 16:16:59,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 4 to mira0435601-251 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 6 to mira0435601-251 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 4 to mira0435601-247 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 80 (ObjNum 246 snap 0 seq_num 246) dirty exists 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 1 to mira0435601-255 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 2 to mira0435601-255 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 3 to mira0435601-255 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 5 to mira0435601-255 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 1 to mira0435601-242 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 3 to mira0435601-242 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 7 to mira0435601-251 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 64 (ObjNum 250 snap 0 seq_num 250) dirty exists 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 6 to mira0435601-255 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 255 v 68 (ObjNum 254 snap 0 seq_num 254) dirty exists 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: done (15 left) 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: done (14 left) 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: done (13 left) 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: write initial oid 261 2014-07-01 16:16:59,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: seq_num 260 ranges {0=561152} 2014-07-01 16:16:59,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: writing mira0435601-261 from 0 to 561152 tid 1 2014-07-01 16:16:59,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: write initial oid 262 2014-07-01 16:16:59,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: seq_num 261 ranges {0=675840,675840=552960,1228800=249856} 2014-07-01 16:16:59,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: writing mira0435601-262 from 0 to 675840 tid 1 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: writing mira0435601-262 from 675840 to 1228800 tid 2 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: writing mira0435601-262 from 1228800 to 1478656 tid 3 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: write initial oid 263 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: seq_num 262 ranges {0=753664,753664=401408} 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: writing mira0435601-263 from 0 to 753664 tid 1 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: writing mira0435601-263 from 753664 to 1155072 tid 2 2014-07-01 16:16:59,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 4 to mira0435601-242 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 56 (ObjNum 241 snap 0 seq_num 241) dirty exists 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 1 to mira0435601-243 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 2 to mira0435601-243 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 3 to mira0435601-243 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 5 to mira0435601-243 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: done (15 left) 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: write initial oid 264 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: seq_num 263 ranges {0=782336,782336=716800,1499136=581632,2080768=307200} 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: writing mira0435601-264 from 0 to 782336 tid 1 2014-07-01 16:16:59,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: writing mira0435601-264 from 782336 to 1499136 tid 2 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: writing mira0435601-264 from 1499136 to 2080768 tid 3 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: writing mira0435601-264 from 2080768 to 2387968 tid 4 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 4 to mira0435601-249 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 1 to mira0435601-245 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 2 to mira0435601-245 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 3 to mira0435601-245 2014-07-01 16:16:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 5 to mira0435601-245 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 1 to mira0435601-246 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 5 to mira0435601-249 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 83 (ObjNum 248 snap 0 seq_num 248) dirty exists 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 2 to mira0435601-246 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 3 to mira0435601-246 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 5 to mira0435601-246 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 6 to mira0435601-243 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 77 (ObjNum 242 snap 0 seq_num 242) dirty exists 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 6 to mira0435601-245 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 245 v 37 (ObjNum 244 snap 0 seq_num 244) dirty exists 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 1 to mira0435601-256 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 1 to mira0435601-254 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 1 to mira0435601-258 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: done (15 left) 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: done (14 left) 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: done (13 left) 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: write initial oid 265 2014-07-01 16:16:59,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: seq_num 264 ranges {0=495616,495616=401408} 2014-07-01 16:16:59,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: writing mira0435601-265 from 0 to 495616 tid 1 2014-07-01 16:16:59,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: writing mira0435601-265 from 495616 to 897024 tid 2 2014-07-01 16:16:59,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: write initial oid 266 2014-07-01 16:16:59,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: seq_num 265 ranges {0=524288,524288=778240,1302528=528384,1830912=299008} 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0435601-266 from 0 to 524288 tid 1 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0435601-266 from 524288 to 1302528 tid 2 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0435601-266 from 1302528 to 1830912 tid 3 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0435601-266 from 1830912 to 2129920 tid 4 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: write initial oid 267 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: seq_num 266 ranges {0=602112,602112=606208,1208320=598016} 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: writing mira0435601-267 from 0 to 602112 tid 1 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: writing mira0435601-267 from 602112 to 1208320 tid 2 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: writing mira0435601-267 from 1208320 to 1806336 tid 3 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 6 to mira0435601-246 2014-07-01 16:16:59,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 60 (ObjNum 245 snap 0 seq_num 245) dirty exists 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 2 to mira0435601-258 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 1 to mira0435601-252 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 2 to mira0435601-256 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 2 to mira0435601-252 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 3 to mira0435601-252 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 2 to mira0435601-254 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 3 to mira0435601-254 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: done (15 left) 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: write initial oid 268 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: seq_num 267 ranges {0=471040,471040=544768,1015808=581632,1597440=77824} 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0435601-268 from 0 to 471040 tid 1 2014-07-01 16:16:59,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0435601-268 from 471040 to 1015808 tid 2 2014-07-01 16:16:59,369.369 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 53 16 329 313 23.6175 76 0.279716 2.69809 2014-07-01 16:16:59,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0435601-268 from 1015808 to 1597440 tid 3 2014-07-01 16:16:59,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0435601-268 from 1597440 to 1675264 tid 4 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 4 to mira0435601-252 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 3 to mira0435601-258 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 6 to mira0435601-252 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 4 to mira0435601-256 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 5 to mira0435601-254 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 1 to mira0435601-257 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 2 to mira0435601-257 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 3 to mira0435601-257 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 4 to mira0435601-257 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 1 to mira0435601-259 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 1 to mira0435601-261 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 6 to mira0435601-257 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 3 to mira0435601-259 2014-07-01 16:16:59,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 1 to mira0435601-260 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 1 to mira0435601-253 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 1 to mira0435601-262 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 3 to mira0435601-261 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 2 to mira0435601-253 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 5 to mira0435601-256 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 256 v 33 (ObjNum 255 snap 0 seq_num 255) dirty exists 2014-07-01 16:16:59,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 3 to mira0435601-253 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 5 to mira0435601-253 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 4 to mira0435601-258 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 6 to mira0435601-258 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 6 to mira0435601-254 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 81 (ObjNum 253 snap 0 seq_num 253) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 7 to mira0435601-258 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 258 v 34 (ObjNum 257 snap 0 seq_num 257) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 4 to mira0435601-261 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 44 (ObjNum 260 snap 0 seq_num 260) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 4 to mira0435601-259 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 85 (ObjNum 258 snap 0 seq_num 258) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 7 to mira0435601-257 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 257 v 42 (ObjNum 256 snap 0 seq_num 256) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 7 to mira0435601-252 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 65 (ObjNum 251 snap 0 seq_num 251) dirty exists 2014-07-01 16:16:59,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 6 to mira0435601-253 2014-07-01 16:16:59,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 69 (ObjNum 252 snap 0 seq_num 252) dirty exists 2014-07-01 16:16:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: done (15 left) 2014-07-01 16:16:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: done (14 left) 2014-07-01 16:16:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: done (13 left) 2014-07-01 16:16:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: done (12 left) 2014-07-01 16:16:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: done (11 left) 2014-07-01 16:16:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: done (10 left) 2014-07-01 16:16:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: done (9 left) 2014-07-01 16:16:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: done (8 left) 2014-07-01 16:16:59,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: write initial oid 269 2014-07-01 16:16:59,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: seq_num 268 ranges {0=745472,745472=774144,1519616=28672} 2014-07-01 16:16:59,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0435601-269 from 0 to 745472 tid 1 2014-07-01 16:16:59,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0435601-269 from 745472 to 1519616 tid 2 2014-07-01 16:16:59,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0435601-269 from 1519616 to 1548288 tid 3 2014-07-01 16:16:59,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: write initial oid 270 2014-07-01 16:16:59,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: seq_num 269 ranges {0=372736} 2014-07-01 16:16:59,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: writing mira0435601-270 from 0 to 372736 tid 1 2014-07-01 16:16:59,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: write initial oid 271 2014-07-01 16:16:59,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: seq_num 270 ranges {0=241664} 2014-07-01 16:16:59,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: writing mira0435601-271 from 0 to 241664 tid 1 2014-07-01 16:16:59,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: write initial oid 272 2014-07-01 16:16:59,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: seq_num 271 ranges {0=565248,565248=401408} 2014-07-01 16:16:59,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: writing mira0435601-272 from 0 to 565248 tid 1 2014-07-01 16:16:59,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: writing mira0435601-272 from 565248 to 966656 tid 2 2014-07-01 16:16:59,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: write initial oid 273 2014-07-01 16:16:59,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: seq_num 272 ranges {0=593920,593920=753664,1347584=700416,2048000=151552} 2014-07-01 16:16:59,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: writing mira0435601-273 from 0 to 593920 tid 1 2014-07-01 16:16:59,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: writing mira0435601-273 from 593920 to 1347584 tid 2 2014-07-01 16:16:59,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: writing mira0435601-273 from 1347584 to 2048000 tid 3 2014-07-01 16:16:59,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: writing mira0435601-273 from 2048000 to 2199552 tid 4 2014-07-01 16:16:59,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: write initial oid 274 2014-07-01 16:16:59,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: seq_num 273 ranges {0=462848,462848=741376,1204224=577536,1781760=286720} 2014-07-01 16:16:59,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0435601-274 from 0 to 462848 tid 1 2014-07-01 16:16:59,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0435601-274 from 462848 to 1204224 tid 2 2014-07-01 16:16:59,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0435601-274 from 1204224 to 1781760 tid 3 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0435601-274 from 1781760 to 2068480 tid 4 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: write initial oid 275 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: seq_num 274 ranges {0=737280,737280=569344,1306624=634880} 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: writing mira0435601-275 from 0 to 737280 tid 1 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: writing mira0435601-275 from 737280 to 1306624 tid 2 2014-07-01 16:16:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: writing mira0435601-275 from 1306624 to 1941504 tid 3 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: write initial oid 276 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: seq_num 275 ranges {0=409600,409600=749568,1159168=454656} 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0435601-276 from 0 to 409600 tid 1 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0435601-276 from 409600 to 1159168 tid 2 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0435601-276 from 1159168 to 1613824 tid 3 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 2 to mira0435601-262 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 3 to mira0435601-262 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 5 to mira0435601-262 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 1 to mira0435601-264 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 1 to mira0435601-263 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 6 to mira0435601-262 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 68 (ObjNum 261 snap 0 seq_num 261) dirty exists 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 1 to mira0435601-265 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 2 to mira0435601-265 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 4 to mira0435601-265 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 2 to mira0435601-264 2014-07-01 16:16:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 1 to mira0435601-270 2014-07-01 16:16:59,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: done (15 left) 2014-07-01 16:16:59,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: write initial oid 277 2014-07-01 16:16:59,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: seq_num 276 ranges {0=684032,684032=581632,1265664=221184} 2014-07-01 16:16:59,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0435601-277 from 0 to 684032 tid 1 2014-07-01 16:16:59,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0435601-277 from 684032 to 1265664 tid 2 2014-07-01 16:16:59,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0435601-277 from 1265664 to 1486848 tid 3 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 3 to mira0435601-270 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 2 to mira0435601-260 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 3 to mira0435601-264 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 4 to mira0435601-264 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 6 to mira0435601-264 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 3 to mira0435601-260 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 5 to mira0435601-260 2014-07-01 16:16:59,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 1 to mira0435601-266 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 7 to mira0435601-264 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 73 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 2 to mira0435601-263 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 4 to mira0435601-263 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 1 to mira0435601-267 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 6 to mira0435601-260 2014-07-01 16:16:59,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 85 (ObjNum 259 snap 0 seq_num 259) dirty exists 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 5 to mira0435601-265 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 265 v 67 (ObjNum 264 snap 0 seq_num 264) dirty exists 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 2 to mira0435601-267 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 3 to mira0435601-267 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 5 to mira0435601-267 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 1 to mira0435601-268 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 2 to mira0435601-268 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 3 to mira0435601-268 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 4 to mira0435601-268 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 6 to mira0435601-268 2014-07-01 16:16:59,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 4 to mira0435601-270 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 270 v 62 (ObjNum 269 snap 0 seq_num 269) dirty exists 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 5 to mira0435601-263 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 48 (ObjNum 262 snap 0 seq_num 262) dirty exists 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 6 to mira0435601-267 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 52 (ObjNum 266 snap 0 seq_num 266) dirty exists 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 7 to mira0435601-268 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 57 (ObjNum 267 snap 0 seq_num 267) dirty exists 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: done (15 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: done (14 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: done (13 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: done (12 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: done (11 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: done (10 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: done (9 left) 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: write initial oid 278 2014-07-01 16:16:59,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: seq_num 277 ranges {0=557056,557056=565248,1122304=237568} 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0435601-278 from 0 to 557056 tid 1 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0435601-278 from 557056 to 1122304 tid 2 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0435601-278 from 1122304 to 1359872 tid 3 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: write initial oid 279 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: seq_num 278 ranges {0=180224} 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: writing mira0435601-279 from 0 to 180224 tid 1 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: write initial oid 280 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: seq_num 279 ranges {0=53248} 2014-07-01 16:16:59,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: writing mira0435601-280 from 0 to 53248 tid 1 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: write initial oid 281 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: seq_num 280 ranges {0=532480,532480=561152,1093632=487424,1581056=557056} 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: writing mira0435601-281 from 0 to 532480 tid 1 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: writing mira0435601-281 from 532480 to 1093632 tid 2 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: writing mira0435601-281 from 1093632 to 1581056 tid 3 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: writing mira0435601-281 from 1581056 to 2138112 tid 4 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: write initial oid 282 2014-07-01 16:16:59,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: seq_num 281 ranges {0=802816,802816=544768,1347584=659456} 2014-07-01 16:16:59,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: writing mira0435601-282 from 0 to 802816 tid 1 2014-07-01 16:16:59,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: writing mira0435601-282 from 802816 to 1347584 tid 2 2014-07-01 16:16:59,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: writing mira0435601-282 from 1347584 to 2007040 tid 3 2014-07-01 16:16:59,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: write initial oid 283 2014-07-01 16:16:59,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: seq_num 282 ranges {0=675840,675840=778240,1454080=425984} 2014-07-01 16:16:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: writing mira0435601-283 from 0 to 675840 tid 1 2014-07-01 16:16:59,950.950 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 26 2014-07-01 16:16:59,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: writing mira0435601-283 from 675840 to 1454080 tid 2 2014-07-01 16:17:00,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: writing mira0435601-283 from 1454080 to 1880064 tid 3 2014-07-01 16:17:00,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: write initial oid 284 2014-07-01 16:17:00,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: seq_num 283 ranges {0=704512} 2014-07-01 16:17:00,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: writing mira0435601-284 from 0 to 704512 tid 1 2014-07-01 16:17:00,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:00,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 1 to mira0435601-273 2014-07-01 16:17:00,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 2 to mira0435601-273 2014-07-01 16:17:00,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 3 to mira0435601-273 2014-07-01 16:17:00,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 4 to mira0435601-273 2014-07-01 16:17:00,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 6 to mira0435601-273 2014-07-01 16:17:00,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 1 to mira0435601-274 2014-07-01 16:17:00,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 2 to mira0435601-274 2014-07-01 16:17:00,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 3 to mira0435601-274 2014-07-01 16:17:00,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 4 to mira0435601-274 2014-07-01 16:17:00,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 6 to mira0435601-274 2014-07-01 16:17:00,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 7 to mira0435601-273 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 38 (ObjNum 272 snap 0 seq_num 272) dirty exists 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: done (15 left) 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: write initial oid 285 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: seq_num 284 ranges {0=573440} 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: writing mira0435601-285 from 0 to 573440 tid 1 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 7 to mira0435601-274 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 43 (ObjNum 273 snap 0 seq_num 273) dirty exists 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: done (15 left) 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: write initial oid 286 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: seq_num 285 ranges {0=495616,495616=528384,1024000=274432} 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0435601-286 from 0 to 495616 tid 1 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0435601-286 from 495616 to 1024000 tid 2 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0435601-286 from 1024000 to 1298432 tid 3 2014-07-01 16:17:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:00,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 54 16 336 320 23.6986 28 0.854613 2.65406 2014-07-01 16:17:01,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 1 to mira0435601-278 2014-07-01 16:17:01,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 2 to mira0435601-278 2014-07-01 16:17:01,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 3 to mira0435601-278 2014-07-01 16:17:01,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 5 to mira0435601-278 2014-07-01 16:17:01,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 1 to mira0435601-280 2014-07-01 16:17:01,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 1 to mira0435601-282 2014-07-01 16:17:01,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 2 to mira0435601-282 2014-07-01 16:17:01,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 3 to mira0435601-280 2014-07-01 16:17:01,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 4 to mira0435601-280 2014-07-01 16:17:01,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 79 (ObjNum 279 snap 0 seq_num 279) dirty exists 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: done (15 left) 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: write initial oid 287 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: seq_num 286 ranges {0=122880} 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: writing mira0435601-287 from 0 to 122880 tid 1 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 6 to mira0435601-278 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 77 (ObjNum 277 snap 0 seq_num 277) dirty exists 2014-07-01 16:17:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: done (15 left) 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: write initial oid 288 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: seq_num 287 ranges {0=794624,794624=536576,1331200=659456,1990656=409600} 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: writing mira0435601-288 from 0 to 794624 tid 1 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: writing mira0435601-288 from 794624 to 1331200 tid 2 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: writing mira0435601-288 from 1331200 to 1990656 tid 3 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: writing mira0435601-288 from 1990656 to 2400256 tid 4 2014-07-01 16:17:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:01,273.273 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 55 16 338 322 23.4131 8 1.38405 2.64631 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 1 to mira0435601-283 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 3 to mira0435601-282 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 5 to mira0435601-282 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 2 to mira0435601-283 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 3 to mira0435601-283 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 1 to mira0435601-284 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 3 to mira0435601-284 2014-07-01 16:17:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 5 to mira0435601-283 2014-07-01 16:17:01,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 4 to mira0435601-284 2014-07-01 16:17:01,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 81 (ObjNum 283 snap 0 seq_num 283) dirty exists 2014-07-01 16:17:01,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: done (15 left) 2014-07-01 16:17:01,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: write initial oid 289 2014-07-01 16:17:01,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: seq_num 288 ranges {0=667648,667648=765952,1433600=536576,1970176=303104} 2014-07-01 16:17:01,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: writing mira0435601-289 from 0 to 667648 tid 1 2014-07-01 16:17:02,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: writing mira0435601-289 from 667648 to 1433600 tid 2 2014-07-01 16:17:02,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: writing mira0435601-289 from 1433600 to 1970176 tid 3 2014-07-01 16:17:02,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: writing mira0435601-289 from 1970176 to 2273280 tid 4 2014-07-01 16:17:02,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:02,272.272 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 56 16 342 326 23.2807 16 2.69432 2.6439 2014-07-01 16:17:02,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 1 to mira0435601-286 2014-07-01 16:17:02,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 2 to mira0435601-286 2014-07-01 16:17:02,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 3 to mira0435601-286 2014-07-01 16:17:02,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 5 to mira0435601-286 2014-07-01 16:17:02,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 1 to mira0435601-288 2014-07-01 16:17:02,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 2 to mira0435601-288 2014-07-01 16:17:02,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 3 to mira0435601-288 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 4 to mira0435601-288 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 6 to mira0435601-288 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 6 to mira0435601-282 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 48 (ObjNum 281 snap 0 seq_num 281) dirty exists 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: done (15 left) 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: write initial oid 290 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: seq_num 289 ranges {0=745472,745472=753664,1499136=450560} 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: writing mira0435601-290 from 0 to 745472 tid 1 2014-07-01 16:17:02,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: writing mira0435601-290 from 745472 to 1499136 tid 2 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: writing mira0435601-290 from 1499136 to 1949696 tid 3 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 6 to mira0435601-283 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 73 (ObjNum 282 snap 0 seq_num 282) dirty exists 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: done (15 left) 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: write initial oid 291 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: seq_num 290 ranges {0=614400,614400=581632,1196032=483328,1679360=139264} 2014-07-01 16:17:02,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0435601-291 from 0 to 614400 tid 1 2014-07-01 16:17:02,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0435601-291 from 614400 to 1196032 tid 2 2014-07-01 16:17:02,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0435601-291 from 1196032 to 1679360 tid 3 2014-07-01 16:17:02,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0435601-291 from 1679360 to 1818624 tid 4 2014-07-01 16:17:02,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:02,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 6 to mira0435601-286 2014-07-01 16:17:02,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 77 (ObjNum 285 snap 0 seq_num 285) dirty exists 2014-07-01 16:17:02,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: done (15 left) 2014-07-01 16:17:02,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: write initial oid 292 2014-07-01 16:17:02,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: seq_num 291 ranges {0=487424,487424=516096,1003520=688128} 2014-07-01 16:17:02,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0435601-292 from 0 to 487424 tid 1 2014-07-01 16:17:02,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0435601-292 from 487424 to 1003520 tid 2 2014-07-01 16:17:02,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0435601-292 from 1003520 to 1691648 tid 3 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 7 to mira0435601-288 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 82 (ObjNum 287 snap 0 seq_num 287) dirty exists 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: done (15 left) 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: write initial oid 293 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: seq_num 292 ranges {0=516096} 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: writing mira0435601-293 from 0 to 516096 tid 1 2014-07-01 16:17:02,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:03,274.274 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 57 16 347 331 23.223 20 0.646985 2.62381 2014-07-01 16:17:04,273.273 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 58 16 347 331 22.8226 0 - 2.62381 2014-07-01 16:17:04,958.958 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:04,958.958 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:04,959.959 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:17:04,959.959 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:17:04,959.959 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:17:04,960.960 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:17:05,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 59 16 347 331 22.4358 0 - 2.62381 2014-07-01 16:17:06,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:17:06.187110min lat: 0.067553 max lat: 14.2279 avg lat: 2.62381 2014-07-01 16:17:06,276.276 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:17:06,276.276 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 60 16 347 331 22.0619 0 - 2.62381 2014-07-01 16:17:06,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 1 to mira0435601-291 2014-07-01 16:17:06,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 2 to mira0435601-291 2014-07-01 16:17:06,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 3 to mira0435601-291 2014-07-01 16:17:06,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 4 to mira0435601-291 2014-07-01 16:17:06,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 6 to mira0435601-291 2014-07-01 16:17:06,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 1 to mira0435601-293 2014-07-01 16:17:06,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 3 to mira0435601-293 2014-07-01 16:17:06,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 1 to mira0435601-292 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 2 to mira0435601-292 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 3 to mira0435601-292 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 5 to mira0435601-292 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 7 to mira0435601-291 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 53 (ObjNum 290 snap 0 seq_num 290) dirty exists 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: done (15 left) 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: write initial oid 294 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: seq_num 293 ranges {0=385024} 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: writing mira0435601-294 from 0 to 385024 tid 1 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 6 to mira0435601-292 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 38 (ObjNum 291 snap 0 seq_num 291) dirty exists 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 4 to mira0435601-293 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 45 (ObjNum 292 snap 0 seq_num 292) dirty exists 2014-07-01 16:17:06,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: done (15 left) 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: done (14 left) 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: write initial oid 295 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: seq_num 294 ranges {0=708608,708608=401408} 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: writing mira0435601-295 from 0 to 708608 tid 1 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: writing mira0435601-295 from 708608 to 1110016 tid 2 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: write initial oid 296 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: seq_num 295 ranges {0=737280,737280=499712,1236992=602112,1839104=503808} 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: writing mira0435601-296 from 0 to 737280 tid 1 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: writing mira0435601-296 from 737280 to 1236992 tid 2 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: writing mira0435601-296 from 1236992 to 1839104 tid 3 2014-07-01 16:17:06,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: writing mira0435601-296 from 1839104 to 2342912 tid 4 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 1 to mira0435601-285 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 3 to mira0435601-285 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 1 to mira0435601-287 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 3 to mira0435601-287 2014-07-01 16:17:06,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 4 to mira0435601-285 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 69 (ObjNum 284 snap 0 seq_num 284) dirty exists 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 4 to mira0435601-287 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 71 (ObjNum 286 snap 0 seq_num 286) dirty exists 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 1 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 2 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 3 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 1 to mira0435601-294 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 3 to mira0435601-294 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 4 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 6 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 7 to mira0435601-289 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 62 (ObjNum 288 snap 0 seq_num 288) dirty exists 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: done (15 left) 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: done (14 left) 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: done (13 left) 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: write initial oid 297 2014-07-01 16:17:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: seq_num 296 ranges {0=606208,606208=729088,1335296=425984,1761280=421888,2183168=28672} 2014-07-01 16:17:06,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0435601-297 from 0 to 606208 tid 1 2014-07-01 16:17:06,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0435601-297 from 606208 to 1335296 tid 2 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0435601-297 from 1335296 to 1761280 tid 3 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0435601-297 from 1761280 to 2183168 tid 4 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0435601-297 from 2183168 to 2211840 tid 5 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: write initial oid 298 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: seq_num 297 ranges {0=479232,479232=557056,1036288=655360,1691648=393216} 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0435601-298 from 0 to 479232 tid 1 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0435601-298 from 479232 to 1036288 tid 2 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0435601-298 from 1036288 to 1691648 tid 3 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0435601-298 from 1691648 to 2084864 tid 4 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: write initial oid 299 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: seq_num 298 ranges {0=552960,552960=790528,1343488=413696} 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: writing mira0435601-299 from 0 to 552960 tid 1 2014-07-01 16:17:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: writing mira0435601-299 from 552960 to 1343488 tid 2 2014-07-01 16:17:07,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: writing mira0435601-299 from 1343488 to 1757184 tid 3 2014-07-01 16:17:07,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:07,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 1 to mira0435601-295 2014-07-01 16:17:07,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 2 to mira0435601-295 2014-07-01 16:17:07,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 1 to mira0435601-297 2014-07-01 16:17:07,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 4 to mira0435601-295 2014-07-01 16:17:07,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 5 to mira0435601-295 2014-07-01 16:17:07,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 56 (ObjNum 294 snap 0 seq_num 294) dirty exists 2014-07-01 16:17:07,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 2 to mira0435601-297 2014-07-01 16:17:07,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 3 to mira0435601-297 2014-07-01 16:17:07,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 4 to mira0435601-297 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 5 to mira0435601-297 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 7 to mira0435601-297 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 8 to mira0435601-297 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 297 v 51 (ObjNum 296 snap 0 seq_num 296) dirty exists 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: done (15 left) 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: done (14 left) 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: write initial oid 300 2014-07-01 16:17:07,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: seq_num 299 ranges {0=425984,425984=724992,1150976=479232} 2014-07-01 16:17:07,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0435601-300 from 0 to 425984 tid 1 2014-07-01 16:17:07,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0435601-300 from 425984 to 1150976 tid 2 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0435601-300 from 1150976 to 1630208 tid 3 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: write initial oid 301 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: seq_num 300 ranges {0=700416,700416=552960,1253376=249856} 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0435601-301 from 0 to 700416 tid 1 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0435601-301 from 700416 to 1253376 tid 2 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0435601-301 from 1253376 to 1503232 tid 3 2014-07-01 16:17:07,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:07,274.274 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 61 15 359 344 22.5525 13 0.158504 2.73181 2014-07-01 16:17:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 1 to mira0435601-300 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 2 to mira0435601-300 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 3 to mira0435601-300 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 5 to mira0435601-300 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 1 to mira0435601-301 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 2 to mira0435601-301 2014-07-01 16:17:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 3 to mira0435601-301 2014-07-01 16:17:08,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 5 to mira0435601-301 2014-07-01 16:17:08,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 6 to mira0435601-300 2014-07-01 16:17:08,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 60 (ObjNum 299 snap 0 seq_num 299) dirty exists 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: done (15 left) 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: write initial oid 302 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: seq_num 301 ranges {0=323584} 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: writing mira0435601-302 from 0 to 323584 tid 1 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 6 to mira0435601-301 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 85 (ObjNum 300 snap 0 seq_num 300) dirty exists 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: done (15 left) 2014-07-01 16:17:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: write initial oid 303 2014-07-01 16:17:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: seq_num 302 ranges {0=196608} 2014-07-01 16:17:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: writing mira0435601-303 from 0 to 196608 tid 1 2014-07-01 16:17:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:08,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 62 16 378 362 23.3498 72 0.457351 2.73663 2014-07-01 16:17:09,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 63 16 388 372 23.6139 40 0.959475 2.68225 2014-07-01 16:17:09,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 4 to mira0435601-294 2014-07-01 16:17:09,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 294 v 84 (ObjNum 293 snap 0 seq_num 293) dirty exists 2014-07-01 16:17:09,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: done (15 left) 2014-07-01 16:17:09,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: write initial oid 304 2014-07-01 16:17:09,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: seq_num 303 ranges {0=520192,520192=401408} 2014-07-01 16:17:09,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: writing mira0435601-304 from 0 to 520192 tid 1 2014-07-01 16:17:09,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: writing mira0435601-304 from 520192 to 921600 tid 2 2014-07-01 16:17:09,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:10,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 64 16 393 377 23.5574 20 1.52598 2.66333 2014-07-01 16:17:10,448.448 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:10,449.449 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:10,449.449 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 16:17:10,449.449 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:17:10,449.449 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:17:10,449.449 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 16:17:11,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 1 to mira0435601-276 2014-07-01 16:17:11,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 2 to mira0435601-276 2014-07-01 16:17:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 3 to mira0435601-276 2014-07-01 16:17:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 5 to mira0435601-276 2014-07-01 16:17:11,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 1 to mira0435601-279 2014-07-01 16:17:11,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 3 to mira0435601-279 2014-07-01 16:17:11,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 2 to mira0435601-266 2014-07-01 16:17:11,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 3 to mira0435601-266 2014-07-01 16:17:11,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 4 to mira0435601-266 2014-07-01 16:17:11,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 6 to mira0435601-266 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 1 to mira0435601-271 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 3 to mira0435601-271 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 4 to mira0435601-279 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 74 (ObjNum 278 snap 0 seq_num 278) dirty exists 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: done (15 left) 2014-07-01 16:17:11,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: write initial oid 305 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: seq_num 304 ranges {0=544768,544768=536576,1081344=475136,1556480=487424,2043904=106496} 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0435601-305 from 0 to 544768 tid 1 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0435601-305 from 544768 to 1081344 tid 2 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0435601-305 from 1081344 to 1556480 tid 3 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0435601-305 from 1556480 to 2043904 tid 4 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0435601-305 from 2043904 to 2150400 tid 5 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 4 to mira0435601-271 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 271 v 92 (ObjNum 270 snap 0 seq_num 270) dirty exists 2014-07-01 16:17:11,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 7 to mira0435601-266 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 90 (ObjNum 265 snap 0 seq_num 265) dirty exists 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 6 to mira0435601-276 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 276 v 72 (ObjNum 275 snap 0 seq_num 275) dirty exists 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 1 to mira0435601-277 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 2 to mira0435601-277 2014-07-01 16:17:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 3 to mira0435601-277 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 5 to mira0435601-277 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 1 to mira0435601-281 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: done (15 left) 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: done (14 left) 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: done (13 left) 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: write initial oid 306 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: seq_num 305 ranges {0=417792,417792=520192,937984=598016,1536000=438272,1974272=49152} 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0435601-306 from 0 to 417792 tid 1 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0435601-306 from 417792 to 937984 tid 2 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0435601-306 from 937984 to 1536000 tid 3 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0435601-306 from 1536000 to 1974272 tid 4 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0435601-306 from 1974272 to 2023424 tid 5 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: write initial oid 307 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: seq_num 306 ranges {0=446464,446464=401408} 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: writing mira0435601-307 from 0 to 446464 tid 1 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: writing mira0435601-307 from 446464 to 847872 tid 2 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: write initial oid 308 2014-07-01 16:17:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: seq_num 307 ranges {0=765952,765952=737280,1503232=65536} 2014-07-01 16:17:11,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: writing mira0435601-308 from 0 to 765952 tid 1 2014-07-01 16:17:11,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: writing mira0435601-308 from 765952 to 1503232 tid 2 2014-07-01 16:17:11,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: writing mira0435601-308 from 1503232 to 1568768 tid 3 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 2 to mira0435601-281 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 3 to mira0435601-281 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 4 to mira0435601-281 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 6 to mira0435601-281 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 7 to mira0435601-281 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 51 (ObjNum 280 snap 0 seq_num 280) dirty exists 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 6 to mira0435601-277 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 46 (ObjNum 276 snap 0 seq_num 276) dirty exists 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 1 to mira0435601-275 2014-07-01 16:17:11,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 2 to mira0435601-275 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 3 to mira0435601-275 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 5 to mira0435601-275 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: done (15 left) 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: done (14 left) 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: write initial oid 309 2014-07-01 16:17:11,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: seq_num 308 ranges {0=638976,638976=516096,1155072=286720} 2014-07-01 16:17:11,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0435601-309 from 0 to 638976 tid 1 2014-07-01 16:17:11,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0435601-309 from 638976 to 1155072 tid 2 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0435601-309 from 1155072 to 1441792 tid 3 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: write initial oid 310 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: seq_num 309 ranges {0=266240} 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: writing mira0435601-310 from 0 to 266240 tid 1 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 1 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 2 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 3 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 4 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 6 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 7 to mira0435601-298 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 53 (ObjNum 297 snap 0 seq_num 297) dirty exists 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: done (15 left) 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: write initial oid 311 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: seq_num 310 ranges {0=135168} 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: writing mira0435601-311 from 0 to 135168 tid 1 2014-07-01 16:17:11,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:11,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 65 16 412 396 24.3639 76 0.6742 2.5936 2014-07-01 16:17:11,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 6 to mira0435601-275 2014-07-01 16:17:11,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 275 v 89 (ObjNum 274 snap 0 seq_num 274) dirty exists 2014-07-01 16:17:11,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: done (15 left) 2014-07-01 16:17:11,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: write initial oid 312 2014-07-01 16:17:11,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: seq_num 311 ranges {0=8192} 2014-07-01 16:17:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: writing mira0435601-312 from 0 to 8192 tid 1 2014-07-01 16:17:11,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:12,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 1 to mira0435601-290 2014-07-01 16:17:12,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 66 16 416 400 24.2372 16 1.33882 2.57826 2014-07-01 16:17:12,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 1 to mira0435601-308 2014-07-01 16:17:12,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 2 to mira0435601-308 2014-07-01 16:17:12,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 3 to mira0435601-308 2014-07-01 16:17:12,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 5 to mira0435601-308 2014-07-01 16:17:13,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 67 16 417 401 23.9351 4 1.3751 2.57526 2014-07-01 16:17:13,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 1 to mira0435601-305 2014-07-01 16:17:13,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 1 to mira0435601-306 2014-07-01 16:17:13,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 2 to mira0435601-305 2014-07-01 16:17:13,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 3 to mira0435601-305 2014-07-01 16:17:13,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 4 to mira0435601-305 2014-07-01 16:17:13,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 5 to mira0435601-305 2014-07-01 16:17:13,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 7 to mira0435601-305 2014-07-01 16:17:13,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 8 to mira0435601-305 2014-07-01 16:17:13,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 90 (ObjNum 304 snap 0 seq_num 304) dirty exists 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: done (15 left) 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: write initial oid 313 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: seq_num 312 ranges {0=487424,487424=495616,983040=614400,1597440=495616} 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: writing mira0435601-313 from 0 to 487424 tid 1 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: writing mira0435601-313 from 487424 to 983040 tid 2 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: writing mira0435601-313 from 983040 to 1597440 tid 3 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: writing mira0435601-313 from 1597440 to 2093056 tid 4 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:13,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 2 to mira0435601-306 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 3 to mira0435601-306 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 4 to mira0435601-306 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 5 to mira0435601-306 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 7 to mira0435601-306 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 1 to mira0435601-309 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 2 to mira0435601-309 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 1 to mira0435601-310 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 3 to mira0435601-309 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 5 to mira0435601-309 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 3 to mira0435601-310 2014-07-01 16:17:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 1 to mira0435601-312 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 3 to mira0435601-312 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 4 to mira0435601-310 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 62 (ObjNum 309 snap 0 seq_num 309) dirty exists 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 4 to mira0435601-312 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 73 (ObjNum 311 snap 0 seq_num 311) dirty exists 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: done (15 left) 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: done (14 left) 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: write initial oid 314 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: seq_num 313 ranges {0=757760,757760=724992,1482752=479232} 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: writing mira0435601-314 from 0 to 757760 tid 1 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: writing mira0435601-314 from 757760 to 1482752 tid 2 2014-07-01 16:17:13,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: writing mira0435601-314 from 1482752 to 1961984 tid 3 2014-07-01 16:17:13,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: write initial oid 315 2014-07-01 16:17:13,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: seq_num 314 ranges {0=630784,630784=712704,1343488=491520} 2014-07-01 16:17:13,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: writing mira0435601-315 from 0 to 630784 tid 1 2014-07-01 16:17:13,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: writing mira0435601-315 from 630784 to 1343488 tid 2 2014-07-01 16:17:13,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: writing mira0435601-315 from 1343488 to 1835008 tid 3 2014-07-01 16:17:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 8 to mira0435601-306 2014-07-01 16:17:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 44 (ObjNum 305 snap 0 seq_num 305) dirty exists 2014-07-01 16:17:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: done (15 left) 2014-07-01 16:17:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: write initial oid 316 2014-07-01 16:17:14,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: seq_num 315 ranges {0=659456} 2014-07-01 16:17:14,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: writing mira0435601-316 from 0 to 659456 tid 1 2014-07-01 16:17:14,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:14,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 6 to mira0435601-308 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 89 (ObjNum 307 snap 0 seq_num 307) dirty exists 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 6 to mira0435601-309 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 48 (ObjNum 308 snap 0 seq_num 308) dirty exists 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: done (15 left) 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: done (14 left) 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: write initial oid 317 2014-07-01 16:17:14,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: seq_num 316 ranges {0=331776} 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: writing mira0435601-317 from 0 to 331776 tid 1 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: write initial oid 318 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: seq_num 317 ranges {0=450560,450560=708608,1159168=94208} 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0435601-318 from 0 to 450560 tid 1 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0435601-318 from 450560 to 1159168 tid 2 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0435601-318 from 1159168 to 1253376 tid 3 2014-07-01 16:17:14,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:14,336.336 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 68 16 420 404 23.7595 12 2.42155 2.5748 2014-07-01 16:17:14,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 1 to mira0435601-303 2014-07-01 16:17:14,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 3 to mira0435601-303 2014-07-01 16:17:14,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 1 to mira0435601-302 2014-07-01 16:17:14,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 2 to mira0435601-290 2014-07-01 16:17:14,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 3 to mira0435601-290 2014-07-01 16:17:14,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 5 to mira0435601-290 2014-07-01 16:17:14,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 1 to mira0435601-296 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 2 to mira0435601-296 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 3 to mira0435601-296 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 4 to mira0435601-296 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 4 to mira0435601-303 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 94 (ObjNum 302 snap 0 seq_num 302) dirty exists 2014-07-01 16:17:15,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: done (15 left) 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: write initial oid 319 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: seq_num 318 ranges {0=73728} 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: writing mira0435601-319 from 0 to 73728 tid 1 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 1 to mira0435601-269 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 2 to mira0435601-269 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 3 to mira0435601-269 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 5 to mira0435601-269 2014-07-01 16:17:15,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 1 to mira0435601-272 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 2 to mira0435601-272 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 4 to mira0435601-272 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 3 to mira0435601-302 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 1 to mira0435601-304 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 5 to mira0435601-272 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 56 (ObjNum 271 snap 0 seq_num 271) dirty exists 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: done (15 left) 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: write initial oid 320 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: seq_num 319 ranges {0=503808,503808=765952,1269760=36864} 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0435601-320 from 0 to 503808 tid 1 2014-07-01 16:17:15,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0435601-320 from 503808 to 1269760 tid 2 2014-07-01 16:17:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0435601-320 from 1269760 to 1306624 tid 3 2014-07-01 16:17:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 6 to mira0435601-269 2014-07-01 16:17:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 269 v 53 (ObjNum 268 snap 0 seq_num 268) dirty exists 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 2 to mira0435601-304 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 4 to mira0435601-304 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 1 to mira0435601-307 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 2 to mira0435601-307 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 6 to mira0435601-296 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 6 to mira0435601-290 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 78 (ObjNum 289 snap 0 seq_num 289) dirty exists 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 4 to mira0435601-302 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 62 (ObjNum 301 snap 0 seq_num 301) dirty exists 2014-07-01 16:17:15,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 5 to mira0435601-304 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 304 v 65 (ObjNum 303 snap 0 seq_num 303) dirty exists 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 7 to mira0435601-296 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 83 (ObjNum 295 snap 0 seq_num 295) dirty exists 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 4 to mira0435601-307 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 1 to mira0435601-311 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 3 to mira0435601-311 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 1 to mira0435601-299 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 2 to mira0435601-299 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: done (15 left) 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: done (14 left) 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: done (13 left) 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: done (12 left) 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: done (11 left) 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: write initial oid 321 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: seq_num 320 ranges {0=622592,622592=704512,1327104=557056,1884160=344064} 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: writing mira0435601-321 from 0 to 622592 tid 1 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: writing mira0435601-321 from 622592 to 1327104 tid 2 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: writing mira0435601-321 from 1327104 to 1884160 tid 3 2014-07-01 16:17:15,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: writing mira0435601-321 from 1884160 to 2228224 tid 4 2014-07-01 16:17:15,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: write initial oid 322 2014-07-01 16:17:15,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: seq_num 321 ranges {0=696320,696320=532480,1228800=671744} 2014-07-01 16:17:15,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: writing mira0435601-322 from 0 to 696320 tid 1 2014-07-01 16:17:15,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: writing mira0435601-322 from 696320 to 1228800 tid 2 2014-07-01 16:17:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: writing mira0435601-322 from 1228800 to 1900544 tid 3 2014-07-01 16:17:15,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: write initial oid 323 2014-07-01 16:17:15,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: seq_num 322 ranges {0=724992} 2014-07-01 16:17:15,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: writing mira0435601-323 from 0 to 724992 tid 1 2014-07-01 16:17:15,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: write initial oid 324 2014-07-01 16:17:15,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: seq_num 323 ranges {0=442368,442368=749568,1191936=454656} 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0435601-324 from 0 to 442368 tid 1 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0435601-324 from 442368 to 1191936 tid 2 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0435601-324 from 1191936 to 1646592 tid 3 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: write initial oid 325 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: seq_num 324 ranges {0=466944} 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: writing mira0435601-325 from 0 to 466944 tid 1 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 5 to mira0435601-307 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 95 (ObjNum 306 snap 0 seq_num 306) dirty exists 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 4 to mira0435601-311 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 67 (ObjNum 310 snap 0 seq_num 310) dirty exists 2014-07-01 16:17:15,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 1 to mira0435601-313 2014-07-01 16:17:15,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 1 to mira0435601-317 2014-07-01 16:17:15,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 3 to mira0435601-299 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 3 to mira0435601-317 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 5 to mira0435601-299 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 4 to mira0435601-317 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 96 (ObjNum 316 snap 0 seq_num 316) dirty exists 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 6 to mira0435601-299 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 93 (ObjNum 298 snap 0 seq_num 298) dirty exists 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 1 to mira0435601-319 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 3 to mira0435601-319 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 2 to mira0435601-313 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 3 to mira0435601-313 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 4 to mira0435601-313 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 6 to mira0435601-313 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 1 to mira0435601-314 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: done (15 left) 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: done (14 left) 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: done (13 left) 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: done (12 left) 2014-07-01 16:17:15,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: write initial oid 326 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: seq_num 325 ranges {0=143360} 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: writing mira0435601-326 from 0 to 143360 tid 1 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: write initial oid 327 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: seq_num 326 ranges {0=663552,663552=401408} 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: writing mira0435601-327 from 0 to 663552 tid 1 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: writing mira0435601-327 from 663552 to 1064960 tid 2 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: write initial oid 328 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: seq_num 327 ranges {0=688128,688128=729088,1417216=729088,2146304=147456} 2014-07-01 16:17:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0435601-328 from 0 to 688128 tid 1 2014-07-01 16:17:15,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 69 16 437 421 24.4005 68 0.860908 2.61404 2014-07-01 16:17:15,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0435601-328 from 688128 to 1417216 tid 2 2014-07-01 16:17:15,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0435601-328 from 1417216 to 2146304 tid 3 2014-07-01 16:17:15,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0435601-328 from 2146304 to 2293760 tid 4 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: write initial oid 329 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 2 to mira0435601-314 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 3 to mira0435601-314 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 5 to mira0435601-314 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 1 to mira0435601-316 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 3 to mira0435601-316 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 1 to mira0435601-320 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 7 to mira0435601-313 2014-07-01 16:17:15,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 88 (ObjNum 312 snap 0 seq_num 312) dirty exists 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 6 to mira0435601-314 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 92 (ObjNum 313 snap 0 seq_num 313) dirty exists 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 4 to mira0435601-319 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 53 (ObjNum 318 snap 0 seq_num 318) dirty exists 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 4 to mira0435601-316 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 94 (ObjNum 315 snap 0 seq_num 315) dirty exists 2014-07-01 16:17:15,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: seq_num 328 ranges {0=716800,716800=401408} 2014-07-01 16:17:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0435601-329 from 0 to 716800 tid 1 2014-07-01 16:17:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0435601-329 from 716800 to 1118208 tid 2 2014-07-01 16:17:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: done (15 left) 2014-07-01 16:17:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: done (14 left) 2014-07-01 16:17:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: done (13 left) 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: done (12 left) 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: write initial oid 330 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: seq_num 329 ranges {0=589824,589824=401408} 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: writing mira0435601-330 from 0 to 589824 tid 1 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: writing mira0435601-330 from 589824 to 991232 tid 2 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: write initial oid 331 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: seq_num 330 ranges {0=507904,507904=724992,1232896=479232} 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: writing mira0435601-331 from 0 to 507904 tid 1 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: writing mira0435601-331 from 507904 to 1232896 tid 2 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: writing mira0435601-331 from 1232896 to 1712128 tid 3 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: write initial oid 332 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: seq_num 331 ranges {0=536576} 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: writing mira0435601-332 from 0 to 536576 tid 1 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: write initial oid 333 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: seq_num 332 ranges {0=409600} 2014-07-01 16:17:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: writing mira0435601-333 from 0 to 409600 tid 1 2014-07-01 16:17:15,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:15,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 1 to mira0435601-321 2014-07-01 16:17:15,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 2 to mira0435601-320 2014-07-01 16:17:15,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 3 to mira0435601-320 2014-07-01 16:17:15,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 5 to mira0435601-320 2014-07-01 16:17:15,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 6 to mira0435601-320 2014-07-01 16:17:15,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 66 (ObjNum 319 snap 0 seq_num 319) dirty exists 2014-07-01 16:17:15,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: done (15 left) 2014-07-01 16:17:15,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: write initial oid 334 2014-07-01 16:17:15,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: seq_num 333 ranges {0=278528} 2014-07-01 16:17:15,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: writing mira0435601-334 from 0 to 278528 tid 1 2014-07-01 16:17:15,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:15,869.869 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:15,869.869 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:15,869.869 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:17:15,869.869 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:17:16,277.277 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 70 16 437 421 24.0519 0 - 2.61404 2014-07-01 16:17:16,620.620 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:17:16,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 1 to mira0435601-315 2014-07-01 16:17:16,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 2 to mira0435601-315 2014-07-01 16:17:16,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 3 to mira0435601-315 2014-07-01 16:17:16,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 5 to mira0435601-315 2014-07-01 16:17:16,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 1 to mira0435601-318 2014-07-01 16:17:16,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 2 to mira0435601-318 2014-07-01 16:17:16,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 3 to mira0435601-318 2014-07-01 16:17:16,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 5 to mira0435601-318 2014-07-01 16:17:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 6 to mira0435601-318 2014-07-01 16:17:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 60 (ObjNum 317 snap 0 seq_num 317) dirty exists 2014-07-01 16:17:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: done (15 left) 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: write initial oid 335 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: seq_num 334 ranges {0=757760,757760=704512,1462272=548864,2011136=352256} 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0435601-335 from 0 to 757760 tid 1 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0435601-335 from 757760 to 1462272 tid 2 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0435601-335 from 1462272 to 2011136 tid 3 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0435601-335 from 2011136 to 2363392 tid 4 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 6 to mira0435601-315 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 60 (ObjNum 314 snap 0 seq_num 314) dirty exists 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: done (15 left) 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: write initial oid 336 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: seq_num 335 ranges {0=630784,630784=536576,1167360=671744,1839104=397312} 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0435601-336 from 0 to 630784 tid 1 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0435601-336 from 630784 to 1167360 tid 2 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0435601-336 from 1167360 to 1839104 tid 3 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0435601-336 from 1839104 to 2236416 tid 4 2014-07-01 16:17:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:17,275.275 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 71 16 442 426 23.9948 10 1.94475 2.60366 2014-07-01 16:17:17,312.312 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:17:17,312.312 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 36' 2014-07-01 16:17:18,277.277 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 72 16 444 428 23.7726 8 0.853501 2.59556 2014-07-01 16:17:18,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 1 to mira0435601-322 2014-07-01 16:17:18,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 2 to mira0435601-322 2014-07-01 16:17:18,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 3 to mira0435601-322 2014-07-01 16:17:18,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 2 to mira0435601-321 2014-07-01 16:17:18,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 5 to mira0435601-322 2014-07-01 16:17:18,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 3 to mira0435601-321 2014-07-01 16:17:18,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 4 to mira0435601-321 2014-07-01 16:17:18,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 1 to mira0435601-327 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 6 to mira0435601-321 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 2 to mira0435601-327 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 4 to mira0435601-327 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 6 to mira0435601-322 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 64 (ObjNum 321 snap 0 seq_num 321) dirty exists 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: done (15 left) 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: write initial oid 337 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: seq_num 336 ranges {0=499712,499712=471040,970752=495616,1466368=438272,1904640=200704} 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0435601-337 from 0 to 499712 tid 1 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0435601-337 from 499712 to 970752 tid 2 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0435601-337 from 970752 to 1466368 tid 3 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0435601-337 from 1466368 to 1904640 tid 4 2014-07-01 16:17:18,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0435601-337 from 1904640 to 2105344 tid 5 2014-07-01 16:17:18,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 7 to mira0435601-321 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 94 (ObjNum 320 snap 0 seq_num 320) dirty exists 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 5 to mira0435601-327 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 77 (ObjNum 326 snap 0 seq_num 326) dirty exists 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: done (15 left) 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: done (14 left) 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: write initial oid 338 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: seq_num 337 ranges {0=774144,774144=700416,1474560=503808} 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: writing mira0435601-338 from 0 to 774144 tid 1 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: writing mira0435601-338 from 774144 to 1474560 tid 2 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: writing mira0435601-338 from 1474560 to 1978368 tid 3 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: write initial oid 339 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: seq_num 338 ranges {0=798720} 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: writing mira0435601-339 from 0 to 798720 tid 1 2014-07-01 16:17:18,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:19,183.183 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 36 2014-07-01 16:17:19,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 1 to mira0435601-324 2014-07-01 16:17:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 2 to mira0435601-324 2014-07-01 16:17:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 3 to mira0435601-324 2014-07-01 16:17:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 5 to mira0435601-324 2014-07-01 16:17:19,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 1 to mira0435601-323 2014-07-01 16:17:19,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 1 to mira0435601-326 2014-07-01 16:17:19,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 3 to mira0435601-326 2014-07-01 16:17:19,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 1 to mira0435601-328 2014-07-01 16:17:19,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 2 to mira0435601-328 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 3 to mira0435601-328 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 4 to mira0435601-328 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 6 to mira0435601-328 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 3 to mira0435601-323 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 1 to mira0435601-333 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 3 to mira0435601-333 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 1 to mira0435601-325 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 3 to mira0435601-325 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 1 to mira0435601-329 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 2 to mira0435601-329 2014-07-01 16:17:19,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 4 to mira0435601-329 2014-07-01 16:17:19,441.441 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 73 16 449 433 23.7209 20 4.30801 2.60992 2014-07-01 16:17:19,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 4 to mira0435601-333 2014-07-01 16:17:19,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 66 (ObjNum 332 snap 0 seq_num 332) dirty exists 2014-07-01 16:17:19,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: done (15 left) 2014-07-01 16:17:19,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: write initial oid 340 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: seq_num 339 ranges {0=720896,720896=516096,1236992=286720} 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: writing mira0435601-340 from 0 to 720896 tid 1 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: writing mira0435601-340 from 720896 to 1236992 tid 2 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: writing mira0435601-340 from 1236992 to 1523712 tid 3 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 7 to mira0435601-328 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 62 (ObjNum 327 snap 0 seq_num 327) dirty exists 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: done (15 left) 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: write initial oid 341 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: seq_num 340 ranges {0=593920,593920=696320,1290240=106496} 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0435601-341 from 0 to 593920 tid 1 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0435601-341 from 593920 to 1290240 tid 2 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0435601-341 from 1290240 to 1396736 tid 3 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:19,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 4 to mira0435601-325 2014-07-01 16:17:19,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 55 (ObjNum 324 snap 0 seq_num 324) dirty exists 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 5 to mira0435601-329 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 329 v 92 (ObjNum 328 snap 0 seq_num 328) dirty exists 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 6 to mira0435601-324 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 71 (ObjNum 323 snap 0 seq_num 323) dirty exists 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 4 to mira0435601-326 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 53 (ObjNum 325 snap 0 seq_num 325) dirty exists 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 4 to mira0435601-323 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 64 (ObjNum 322 snap 0 seq_num 322) dirty exists 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 1 to mira0435601-332 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 3 to mira0435601-332 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 1 to mira0435601-330 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 2 to mira0435601-330 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: done (15 left) 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: done (14 left) 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: done (13 left) 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: done (12 left) 2014-07-01 16:17:19,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: done (11 left) 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: write initial oid 342 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: seq_num 341 ranges {0=217088} 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: writing mira0435601-342 from 0 to 217088 tid 1 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: write initial oid 343 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: seq_num 342 ranges {0=647168,647168=512000,1159168=290816} 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: writing mira0435601-343 from 0 to 647168 tid 1 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: writing mira0435601-343 from 647168 to 1159168 tid 2 2014-07-01 16:17:19,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: writing mira0435601-343 from 1159168 to 1449984 tid 3 2014-07-01 16:17:19,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: write initial oid 344 2014-07-01 16:17:19,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: seq_num 343 ranges {0=413696,413696=401408} 2014-07-01 16:17:19,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: writing mira0435601-344 from 0 to 413696 tid 1 2014-07-01 16:17:19,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: writing mira0435601-344 from 413696 to 815104 tid 2 2014-07-01 16:17:19,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: write initial oid 345 2014-07-01 16:17:19,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: seq_num 344 ranges {0=438272,438272=675840,1114112=790528,1904640=139264} 2014-07-01 16:17:19,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: writing mira0435601-345 from 0 to 438272 tid 1 2014-07-01 16:17:19,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: writing mira0435601-345 from 438272 to 1114112 tid 2 2014-07-01 16:17:19,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: writing mira0435601-345 from 1114112 to 1904640 tid 3 2014-07-01 16:17:19,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: writing mira0435601-345 from 1904640 to 2043904 tid 4 2014-07-01 16:17:19,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: write initial oid 346 2014-07-01 16:17:19,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: seq_num 345 ranges {0=466944,466944=401408} 2014-07-01 16:17:19,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: writing mira0435601-346 from 0 to 466944 tid 1 2014-07-01 16:17:19,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: writing mira0435601-346 from 466944 to 868352 tid 2 2014-07-01 16:17:19,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:19,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 1 to mira0435601-331 2014-07-01 16:17:19,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 4 to mira0435601-330 2014-07-01 16:17:19,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 2 to mira0435601-331 2014-07-01 16:17:19,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 1 to mira0435601-334 2014-07-01 16:17:19,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 3 to mira0435601-331 2014-07-01 16:17:19,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 5 to mira0435601-331 2014-07-01 16:17:19,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 3 to mira0435601-334 2014-07-01 16:17:19,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 5 to mira0435601-330 2014-07-01 16:17:19,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 65 (ObjNum 329 snap 0 seq_num 329) dirty exists 2014-07-01 16:17:19,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 4 to mira0435601-332 2014-07-01 16:17:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 53 (ObjNum 331 snap 0 seq_num 331) dirty exists 2014-07-01 16:17:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 6 to mira0435601-331 2014-07-01 16:17:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 52 (ObjNum 330 snap 0 seq_num 330) dirty exists 2014-07-01 16:17:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 4 to mira0435601-334 2014-07-01 16:17:19,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 50 (ObjNum 333 snap 0 seq_num 333) dirty exists 2014-07-01 16:17:19,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: done (15 left) 2014-07-01 16:17:19,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: done (14 left) 2014-07-01 16:17:19,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: done (13 left) 2014-07-01 16:17:19,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: done (12 left) 2014-07-01 16:17:19,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: write initial oid 347 2014-07-01 16:17:19,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: seq_num 346 ranges {0=585728,585728=491520,1077248=712704} 2014-07-01 16:17:19,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: writing mira0435601-347 from 0 to 585728 tid 1 2014-07-01 16:17:19,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: writing mira0435601-347 from 585728 to 1077248 tid 2 2014-07-01 16:17:19,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: writing mira0435601-347 from 1077248 to 1789952 tid 3 2014-07-01 16:17:19,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: write initial oid 348 2014-07-01 16:17:19,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: seq_num 347 ranges {0=610304} 2014-07-01 16:17:19,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: writing mira0435601-348 from 0 to 610304 tid 1 2014-07-01 16:17:19,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: write initial oid 349 2014-07-01 16:17:19,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: seq_num 348 ranges {0=286720} 2014-07-01 16:17:19,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: writing mira0435601-349 from 0 to 286720 tid 1 2014-07-01 16:17:19,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: write initial oid 350 2014-07-01 16:17:19,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: seq_num 349 ranges {0=405504,405504=487424,892928=315392} 2014-07-01 16:17:19,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0435601-350 from 0 to 405504 tid 1 2014-07-01 16:17:19,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0435601-350 from 405504 to 892928 tid 2 2014-07-01 16:17:19,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0435601-350 from 892928 to 1208320 tid 3 2014-07-01 16:17:19,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:20,276.276 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 74 16 456 440 23.7786 28 0.306937 2.61105 2014-07-01 16:17:21,278.278 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 75 16 457 441 23.5149 4 1.52559 2.60859 2014-07-01 16:17:21,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 1 to mira0435601-343 2014-07-01 16:17:21,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 1 to mira0435601-335 2014-07-01 16:17:21,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 2 to mira0435601-335 2014-07-01 16:17:21,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 3 to mira0435601-335 2014-07-01 16:17:21,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 4 to mira0435601-335 2014-07-01 16:17:21,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 2 to mira0435601-343 2014-07-01 16:17:21,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 6 to mira0435601-335 2014-07-01 16:17:21,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 3 to mira0435601-343 2014-07-01 16:17:21,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 5 to mira0435601-343 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 1 to mira0435601-348 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 3 to mira0435601-348 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 1 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 2 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 3 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 4 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 6 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 7 to mira0435601-335 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 71 (ObjNum 334 snap 0 seq_num 334) dirty exists 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: done (15 left) 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: write initial oid 351 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: seq_num 350 ranges {0=28672} 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: writing mira0435601-351 from 0 to 28672 tid 1 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 7 to mira0435601-336 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 60 (ObjNum 335 snap 0 seq_num 335) dirty exists 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: done (15 left) 2014-07-01 16:17:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: write initial oid 352 2014-07-01 16:17:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: seq_num 351 ranges {0=458752,458752=548864,1007616=253952} 2014-07-01 16:17:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0435601-352 from 0 to 458752 tid 1 2014-07-01 16:17:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0435601-352 from 458752 to 1007616 tid 2 2014-07-01 16:17:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0435601-352 from 1007616 to 1261568 tid 3 2014-07-01 16:17:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:21,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 1 to mira0435601-338 2014-07-01 16:17:21,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 1 to mira0435601-337 2014-07-01 16:17:21,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 2 to mira0435601-338 2014-07-01 16:17:21,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 2 to mira0435601-337 2014-07-01 16:17:21,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 1 to mira0435601-339 2014-07-01 16:17:21,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 3 to mira0435601-337 2014-07-01 16:17:21,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 4 to mira0435601-337 2014-07-01 16:17:22,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 5 to mira0435601-337 2014-07-01 16:17:22,277.277 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 76 16 459 443 23.3107 8 3.67761 2.60963 2014-07-01 16:17:23,278.278 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 77 16 459 443 23.008 0 - 2.60963 2014-07-01 16:17:23,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 3 to mira0435601-339 2014-07-01 16:17:23,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 1 to mira0435601-341 2014-07-01 16:17:23,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 2 to mira0435601-341 2014-07-01 16:17:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 3 to mira0435601-341 2014-07-01 16:17:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 5 to mira0435601-341 2014-07-01 16:17:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 3 to mira0435601-338 2014-07-01 16:17:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 5 to mira0435601-338 2014-07-01 16:17:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 7 to mira0435601-337 2014-07-01 16:17:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 1 to mira0435601-340 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 2 to mira0435601-340 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 3 to mira0435601-340 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 1 to mira0435601-347 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 5 to mira0435601-340 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 1 to mira0435601-342 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 3 to mira0435601-342 2014-07-01 16:17:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 1 to mira0435601-345 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 2 to mira0435601-345 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 1 to mira0435601-344 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 8 to mira0435601-337 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 96 (ObjNum 336 snap 0 seq_num 336) dirty exists 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 2 to mira0435601-344 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: done (15 left) 2014-07-01 16:17:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: write initial oid 353 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: seq_num 352 ranges {0=626688} 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: writing mira0435601-353 from 0 to 626688 tid 1 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 2 to mira0435601-347 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 4 to mira0435601-344 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 1 to mira0435601-351 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 3 to mira0435601-351 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 3 to mira0435601-347 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 5 to mira0435601-347 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 1 to mira0435601-350 2014-07-01 16:17:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 2 to mira0435601-350 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 6 to mira0435601-338 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 56 (ObjNum 337 snap 0 seq_num 337) dirty exists 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 4 to mira0435601-339 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 95 (ObjNum 338 snap 0 seq_num 338) dirty exists 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: done (15 left) 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: done (14 left) 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: write initial oid 354 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: seq_num 353 ranges {0=651264,651264=712704,1363968=491520} 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: writing mira0435601-354 from 0 to 651264 tid 1 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: writing mira0435601-354 from 651264 to 1363968 tid 2 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: writing mira0435601-354 from 1363968 to 1855488 tid 3 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: write initial oid 355 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: seq_num 354 ranges {0=679936} 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: writing mira0435601-355 from 0 to 679936 tid 1 2014-07-01 16:17:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:23,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 3 to mira0435601-345 2014-07-01 16:17:23,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 3 to mira0435601-350 2014-07-01 16:17:23,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 5 to mira0435601-350 2014-07-01 16:17:23,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 6 to mira0435601-343 2014-07-01 16:17:23,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 70 (ObjNum 342 snap 0 seq_num 342) dirty exists 2014-07-01 16:17:23,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: done (15 left) 2014-07-01 16:17:23,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: write initial oid 356 2014-07-01 16:17:23,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: seq_num 355 ranges {0=552960} 2014-07-01 16:17:24,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: writing mira0435601-356 from 0 to 552960 tid 1 2014-07-01 16:17:24,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 4 to mira0435601-348 2014-07-01 16:17:24,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 72 (ObjNum 347 snap 0 seq_num 347) dirty exists 2014-07-01 16:17:24,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 1 to mira0435601-346 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 4 to mira0435601-345 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: done (15 left) 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: write initial oid 357 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: seq_num 356 ranges {0=421888} 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: writing mira0435601-357 from 0 to 421888 tid 1 2014-07-01 16:17:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 6 to mira0435601-345 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 2 to mira0435601-346 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 6 to mira0435601-340 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 75 (ObjNum 339 snap 0 seq_num 339) dirty exists 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 4 to mira0435601-346 2014-07-01 16:17:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 1 to mira0435601-354 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 1 to mira0435601-353 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 3 to mira0435601-353 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 1 to mira0435601-355 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 3 to mira0435601-355 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: done (15 left) 2014-07-01 16:17:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: write initial oid 358 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: seq_num 357 ranges {0=98304} 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: writing mira0435601-358 from 0 to 98304 tid 1 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 2 to mira0435601-354 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 3 to mira0435601-354 2014-07-01 16:17:24,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 5 to mira0435601-354 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 1 to mira0435601-349 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 6 to mira0435601-347 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 69 (ObjNum 346 snap 0 seq_num 346) dirty exists 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: done (15 left) 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: write initial oid 359 2014-07-01 16:17:24,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: seq_num 358 ranges {0=774144,774144=679936,1454080=675840,2129920=249856} 2014-07-01 16:17:24,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0435601-359 from 0 to 774144 tid 1 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0435601-359 from 774144 to 1454080 tid 2 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0435601-359 from 1454080 to 2129920 tid 3 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0435601-359 from 2129920 to 2379776 tid 4 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 3 to mira0435601-349 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 1 to mira0435601-352 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 6 to mira0435601-341 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 99 (ObjNum 340 snap 0 seq_num 340) dirty exists 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 1 to mira0435601-356 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 3 to mira0435601-356 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 1 to mira0435601-357 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 3 to mira0435601-357 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 5 to mira0435601-344 2014-07-01 16:17:24,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 99 (ObjNum 343 snap 0 seq_num 343) dirty exists 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 6 to mira0435601-350 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 60 (ObjNum 349 snap 0 seq_num 349) dirty exists 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 2 to mira0435601-352 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 3 to mira0435601-352 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 4 to mira0435601-351 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 101 (ObjNum 350 snap 0 seq_num 350) dirty exists 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 5 to mira0435601-352 2014-07-01 16:17:24,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: done (15 left) 2014-07-01 16:17:24,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: done (14 left) 2014-07-01 16:17:24,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: done (13 left) 2014-07-01 16:17:24,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: done (12 left) 2014-07-01 16:17:24,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: write initial oid 360 2014-07-01 16:17:24,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: seq_num 359 ranges {0=643072,643072=507904,1150976=503808,1654784=421888,2076672=172032} 2014-07-01 16:17:24,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0435601-360 from 0 to 643072 tid 1 2014-07-01 16:17:24,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0435601-360 from 643072 to 1150976 tid 2 2014-07-01 16:17:24,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0435601-360 from 1150976 to 1654784 tid 3 2014-07-01 16:17:24,190.190 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:24,190.190 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:24,190.190 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_1 2014-07-01 16:17:24,191.191 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:17:24,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0435601-360 from 1654784 to 2076672 tid 4 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0435601-360 from 2076672 to 2248704 tid 5 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: write initial oid 361 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: seq_num 360 ranges {0=671744,671744=401408} 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0435601-361 from 0 to 671744 tid 1 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0435601-361 from 671744 to 1073152 tid 2 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: write initial oid 362 2014-07-01 16:17:24,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: seq_num 361 ranges {0=589824,589824=675840,1265664=450560,1716224=77824} 2014-07-01 16:17:24,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: writing mira0435601-362 from 0 to 589824 tid 1 2014-07-01 16:17:24,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: writing mira0435601-362 from 589824 to 1265664 tid 2 2014-07-01 16:17:24,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: writing mira0435601-362 from 1265664 to 1716224 tid 3 2014-07-01 16:17:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: writing mira0435601-362 from 1716224 to 1794048 tid 4 2014-07-01 16:17:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: write initial oid 363 2014-07-01 16:17:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: seq_num 362 ranges {0=462848,462848=503808,966656=700416} 2014-07-01 16:17:24,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: writing mira0435601-363 from 0 to 462848 tid 1 2014-07-01 16:17:24,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: writing mira0435601-363 from 462848 to 966656 tid 2 2014-07-01 16:17:24,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 78 16 477 461 23.6359 36 2.93338 2.67301 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: writing mira0435601-363 from 966656 to 1667072 tid 3 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 5 to mira0435601-346 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 74 (ObjNum 345 snap 0 seq_num 345) dirty exists 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 1 to mira0435601-358 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 3 to mira0435601-358 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 4 to mira0435601-353 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 66 (ObjNum 352 snap 0 seq_num 352) dirty exists 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 4 to mira0435601-349 2014-07-01 16:17:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 55 (ObjNum 348 snap 0 seq_num 348) dirty exists 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 6 to mira0435601-354 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 354 v 99 (ObjNum 353 snap 0 seq_num 353) dirty exists 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 1 to mira0435601-359 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 2 to mira0435601-359 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 3 to mira0435601-359 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 4 to mira0435601-359 2014-07-01 16:17:24,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 6 to mira0435601-359 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 4 to mira0435601-356 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 356 v 55 (ObjNum 355 snap 0 seq_num 355) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 4 to mira0435601-357 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 103 (ObjNum 356 snap 0 seq_num 356) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 4 to mira0435601-358 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 98 (ObjNum 357 snap 0 seq_num 357) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 6 to mira0435601-352 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 352 v 59 (ObjNum 351 snap 0 seq_num 351) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 4 to mira0435601-342 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 69 (ObjNum 341 snap 0 seq_num 341) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 7 to mira0435601-345 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 345 v 74 (ObjNum 344 snap 0 seq_num 344) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 7 to mira0435601-359 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 104 (ObjNum 358 snap 0 seq_num 358) dirty exists 2014-07-01 16:17:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 1 to mira0435601-361 2014-07-01 16:17:24,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 2 to mira0435601-361 2014-07-01 16:17:24,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 4 to mira0435601-361 2014-07-01 16:17:24,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 5 to mira0435601-361 2014-07-01 16:17:24,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 361 v 75 (ObjNum 360 snap 0 seq_num 360) dirty exists 2014-07-01 16:17:24,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 1 to mira0435601-363 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: done (15 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: done (14 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: done (13 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: done (12 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: done (11 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: done (10 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: done (9 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: done (8 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: done (7 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: done (6 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: done (5 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: done (4 left) 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: write initial oid 364 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: seq_num 363 ranges {0=491520} 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: writing mira0435601-364 from 0 to 491520 tid 1 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: write initial oid 365 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: seq_num 364 ranges {0=360448} 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: writing mira0435601-365 from 0 to 360448 tid 1 2014-07-01 16:17:24,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: write initial oid 366 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: seq_num 365 ranges {0=233472} 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: writing mira0435601-366 from 0 to 233472 tid 1 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: write initial oid 367 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: seq_num 366 ranges {0=712704,712704=483328,1196032=569344,1765376=552960} 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0435601-367 from 0 to 712704 tid 1 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0435601-367 from 712704 to 1196032 tid 2 2014-07-01 16:17:24,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0435601-367 from 1196032 to 1765376 tid 3 2014-07-01 16:17:24,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0435601-367 from 1765376 to 2318336 tid 4 2014-07-01 16:17:24,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: write initial oid 368 2014-07-01 16:17:24,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: seq_num 367 ranges {0=581632,581632=716800,1298432=446464,1744896=442368} 2014-07-01 16:17:24,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0435601-368 from 0 to 581632 tid 1 2014-07-01 16:17:24,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0435601-368 from 581632 to 1298432 tid 2 2014-07-01 16:17:24,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0435601-368 from 1298432 to 1744896 tid 3 2014-07-01 16:17:24,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0435601-368 from 1744896 to 2187264 tid 4 2014-07-01 16:17:24,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: write initial oid 369 2014-07-01 16:17:24,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: seq_num 368 ranges {0=610304,610304=401408} 2014-07-01 16:17:24,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: writing mira0435601-369 from 0 to 610304 tid 1 2014-07-01 16:17:24,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: writing mira0435601-369 from 610304 to 1011712 tid 2 2014-07-01 16:17:24,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: write initial oid 370 2014-07-01 16:17:24,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: seq_num 369 ranges {0=483328,483328=401408} 2014-07-01 16:17:24,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: writing mira0435601-370 from 0 to 483328 tid 1 2014-07-01 16:17:24,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: writing mira0435601-370 from 483328 to 884736 tid 2 2014-07-01 16:17:24,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: write initial oid 371 2014-07-01 16:17:24,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: seq_num 370 ranges {0=802816,802816=466944,1269760=335872} 2014-07-01 16:17:24,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: writing mira0435601-371 from 0 to 802816 tid 1 2014-07-01 16:17:24,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: writing mira0435601-371 from 802816 to 1269760 tid 2 2014-07-01 16:17:24,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: writing mira0435601-371 from 1269760 to 1605632 tid 3 2014-07-01 16:17:24,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: write initial oid 372 2014-07-01 16:17:24,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: seq_num 371 ranges {0=430080} 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: writing mira0435601-372 from 0 to 430080 tid 1 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: write initial oid 373 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: seq_num 372 ranges {0=303104} 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: writing mira0435601-373 from 0 to 303104 tid 1 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: write initial oid 374 2014-07-01 16:17:24,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: seq_num 373 ranges {0=172032} 2014-07-01 16:17:24,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: writing mira0435601-374 from 0 to 172032 tid 1 2014-07-01 16:17:24,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: write initial oid 375 2014-07-01 16:17:24,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: seq_num 374 ranges {0=602112,602112=692224,1294336=110592} 2014-07-01 16:17:24,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: writing mira0435601-375 from 0 to 602112 tid 1 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: writing mira0435601-375 from 602112 to 1294336 tid 2 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: writing mira0435601-375 from 1294336 to 1404928 tid 3 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 2 to mira0435601-363 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 1 to mira0435601-362 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 2 to mira0435601-362 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 1 to mira0435601-365 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 3 to mira0435601-365 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 3 to mira0435601-362 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 4 to mira0435601-365 2014-07-01 16:17:24,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 365 v 62 (ObjNum 364 snap 0 seq_num 364) dirty exists 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 4 to mira0435601-362 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 4 to mira0435601-355 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 62 (ObjNum 354 snap 0 seq_num 354) dirty exists 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: done (15 left) 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: done (14 left) 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: write initial oid 376 2014-07-01 16:17:24,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: seq_num 375 ranges {0=524288,524288=520192,1044480=737280,1781760=348160} 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0435601-376 from 0 to 524288 tid 1 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0435601-376 from 524288 to 1044480 tid 2 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0435601-376 from 1044480 to 1781760 tid 3 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0435601-376 from 1781760 to 2129920 tid 4 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: write initial oid 377 2014-07-01 16:17:24,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: seq_num 376 ranges {0=794624,794624=458752,1253376=409600,1662976=335872} 2014-07-01 16:17:24,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: writing mira0435601-377 from 0 to 794624 tid 1 2014-07-01 16:17:24,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: writing mira0435601-377 from 794624 to 1253376 tid 2 2014-07-01 16:17:24,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: writing mira0435601-377 from 1253376 to 1662976 tid 3 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: writing mira0435601-377 from 1662976 to 1998848 tid 4 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 6 to mira0435601-362 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 7 to mira0435601-362 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 67 (ObjNum 361 snap 0 seq_num 361) dirty exists 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: done (15 left) 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: write initial oid 378 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: seq_num 377 ranges {0=421888,421888=401408} 2014-07-01 16:17:24,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: writing mira0435601-378 from 0 to 421888 tid 1 2014-07-01 16:17:24,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: writing mira0435601-378 from 421888 to 823296 tid 2 2014-07-01 16:17:24,800.800 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:17:24,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 1 to mira0435601-360 2014-07-01 16:17:25,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 2 to mira0435601-360 2014-07-01 16:17:25,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 3 to mira0435601-360 2014-07-01 16:17:25,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 4 to mira0435601-360 2014-07-01 16:17:25,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 5 to mira0435601-360 2014-07-01 16:17:25,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 7 to mira0435601-360 2014-07-01 16:17:25,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 8 to mira0435601-360 2014-07-01 16:17:25,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 59 (ObjNum 359 snap 0 seq_num 359) dirty exists 2014-07-01 16:17:25,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: done (15 left) 2014-07-01 16:17:25,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: write initial oid 379 2014-07-01 16:17:25,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: seq_num 378 ranges {0=696320} 2014-07-01 16:17:25,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: writing mira0435601-379 from 0 to 696320 tid 1 2014-07-01 16:17:25,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 1 to mira0435601-366 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 3 to mira0435601-366 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 4 to mira0435601-366 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 76 (ObjNum 365 snap 0 seq_num 365) dirty exists 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: done (15 left) 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: write initial oid 380 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: seq_num 379 ranges {0=614400,614400=503808,1118208=299008} 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: writing mira0435601-380 from 0 to 614400 tid 1 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: writing mira0435601-380 from 614400 to 1118208 tid 2 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: writing mira0435601-380 from 1118208 to 1417216 tid 3 2014-07-01 16:17:25,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,278.278 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 79 16 483 467 23.6404 24 0.651261 2.65533 2014-07-01 16:17:25,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 1 to mira0435601-364 2014-07-01 16:17:25,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 3 to mira0435601-364 2014-07-01 16:17:25,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 1 to mira0435601-367 2014-07-01 16:17:25,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 3 to mira0435601-363 2014-07-01 16:17:25,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 2 to mira0435601-367 2014-07-01 16:17:25,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 5 to mira0435601-363 2014-07-01 16:17:25,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 3 to mira0435601-367 2014-07-01 16:17:25,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 4 to mira0435601-367 2014-07-01 16:17:25,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 6 to mira0435601-367 2014-07-01 16:17:25,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 6 to mira0435601-363 2014-07-01 16:17:25,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 79 (ObjNum 362 snap 0 seq_num 362) dirty exists 2014-07-01 16:17:25,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: done (15 left) 2014-07-01 16:17:25,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: write initial oid 381 2014-07-01 16:17:25,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: seq_num 380 ranges {0=241664} 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: writing mira0435601-381 from 0 to 241664 tid 1 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 4 to mira0435601-364 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 364 v 62 (ObjNum 363 snap 0 seq_num 363) dirty exists 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: done (15 left) 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: write initial oid 382 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: seq_num 381 ranges {0=110592} 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: writing mira0435601-382 from 0 to 110592 tid 1 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 7 to mira0435601-367 2014-07-01 16:17:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 82 (ObjNum 366 snap 0 seq_num 366) dirty exists 2014-07-01 16:17:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: done (15 left) 2014-07-01 16:17:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: write initial oid 383 2014-07-01 16:17:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: seq_num 382 ranges {0=786432,786432=655360,1441792=405504,1847296=544768} 2014-07-01 16:17:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0435601-383 from 0 to 786432 tid 1 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0435601-383 from 786432 to 1441792 tid 2 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0435601-383 from 1441792 to 1847296 tid 3 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0435601-383 from 1847296 to 2392064 tid 4 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 1 to mira0435601-369 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 1 to mira0435601-368 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 2 to mira0435601-368 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 3 to mira0435601-368 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 4 to mira0435601-368 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 6 to mira0435601-368 2014-07-01 16:17:25,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 7 to mira0435601-368 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 65 (ObjNum 367 snap 0 seq_num 367) dirty exists 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: done (15 left) 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: write initial oid 384 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: seq_num 383 ranges {0=413696,413696=729088,1142784=73728} 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0435601-384 from 0 to 413696 tid 1 2014-07-01 16:17:25,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0435601-384 from 413696 to 1142784 tid 2 2014-07-01 16:17:25,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0435601-384 from 1142784 to 1216512 tid 3 2014-07-01 16:17:25,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 2 to mira0435601-369 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 4 to mira0435601-369 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 1 to mira0435601-372 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 3 to mira0435601-372 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 1 to mira0435601-371 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 1 to mira0435601-374 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 3 to mira0435601-374 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 1 to mira0435601-376 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 2 to mira0435601-371 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 2 to mira0435601-376 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 3 to mira0435601-371 2014-07-01 16:17:25,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 1 to mira0435601-375 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 5 to mira0435601-371 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 2 to mira0435601-375 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 3 to mira0435601-375 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 1 to mira0435601-373 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 5 to mira0435601-375 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 3 to mira0435601-373 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 1 to mira0435601-370 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 2 to mira0435601-370 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 4 to mira0435601-370 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 1 to mira0435601-381 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 3 to mira0435601-381 2014-07-01 16:17:25,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 5 to mira0435601-369 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 369 v 70 (ObjNum 368 snap 0 seq_num 368) dirty exists 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 1 to mira0435601-379 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 3 to mira0435601-379 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 4 to mira0435601-372 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 76 (ObjNum 371 snap 0 seq_num 371) dirty exists 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 4 to mira0435601-379 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 81 (ObjNum 378 snap 0 seq_num 378) dirty exists 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 4 to mira0435601-374 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 61 (ObjNum 373 snap 0 seq_num 373) dirty exists 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 6 to mira0435601-371 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 107 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 5 to mira0435601-370 2014-07-01 16:17:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 65 (ObjNum 369 snap 0 seq_num 369) dirty exists 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 4 to mira0435601-381 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 84 (ObjNum 380 snap 0 seq_num 380) dirty exists 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: done (15 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: done (14 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: done (13 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: done (12 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: done (11 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: done (10 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: done (9 left) 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: write initial oid 385 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: seq_num 384 ranges {0=733184,733184=712704,1445888=458752,1904640=32768} 2014-07-01 16:17:25,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: writing mira0435601-385 from 0 to 733184 tid 1 2014-07-01 16:17:25,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: writing mira0435601-385 from 733184 to 1445888 tid 2 2014-07-01 16:17:25,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: writing mira0435601-385 from 1445888 to 1904640 tid 3 2014-07-01 16:17:25,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: writing mira0435601-385 from 1904640 to 1937408 tid 4 2014-07-01 16:17:25,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: write initial oid 386 2014-07-01 16:17:25,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: seq_num 385 ranges {0=606208,606208=495616,1101824=577536,1679360=131072} 2014-07-01 16:17:25,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: writing mira0435601-386 from 0 to 606208 tid 1 2014-07-01 16:17:25,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: writing mira0435601-386 from 606208 to 1101824 tid 2 2014-07-01 16:17:25,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: writing mira0435601-386 from 1101824 to 1679360 tid 3 2014-07-01 16:17:25,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: writing mira0435601-386 from 1679360 to 1810432 tid 4 2014-07-01 16:17:25,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: write initial oid 387 2014-07-01 16:17:25,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: seq_num 386 ranges {0=634880} 2014-07-01 16:17:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: writing mira0435601-387 from 0 to 634880 tid 1 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: write initial oid 388 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 6 to mira0435601-375 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 375 v 69 (ObjNum 374 snap 0 seq_num 374) dirty exists 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 4 to mira0435601-373 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 71 (ObjNum 372 snap 0 seq_num 372) dirty exists 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 3 to mira0435601-376 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 4 to mira0435601-376 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 6 to mira0435601-376 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 1 to mira0435601-378 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 2 to mira0435601-378 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 4 to mira0435601-378 2014-07-01 16:17:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: seq_num 387 ranges {0=503808} 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: writing mira0435601-388 from 0 to 503808 tid 1 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: done (12 left) 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: done (11 left) 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: write initial oid 389 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: seq_num 388 ranges {0=425984,425984=696320,1122304=106496} 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0435601-389 from 0 to 425984 tid 1 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0435601-389 from 425984 to 1122304 tid 2 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0435601-389 from 1122304 to 1228800 tid 3 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: write initial oid 390 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: seq_num 389 ranges {0=53248} 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: writing mira0435601-390 from 0 to 53248 tid 1 2014-07-01 16:17:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: write initial oid 391 2014-07-01 16:17:25,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: seq_num 390 ranges {0=724992,724992=458752,1183744=450560,1634304=421888,2056192=274432} 2014-07-01 16:17:25,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0435601-391 from 0 to 724992 tid 1 2014-07-01 16:17:25,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0435601-391 from 724992 to 1183744 tid 2 2014-07-01 16:17:25,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0435601-391 from 1183744 to 1634304 tid 3 2014-07-01 16:17:25,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0435601-391 from 1634304 to 2056192 tid 4 2014-07-01 16:17:25,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0435601-391 from 2056192 to 2330624 tid 5 2014-07-01 16:17:25,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: write initial oid 392 2014-07-01 16:17:25,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: seq_num 391 ranges {0=753664,753664=401408} 2014-07-01 16:17:25,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: writing mira0435601-392 from 0 to 753664 tid 1 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: writing mira0435601-392 from 753664 to 1155072 tid 2 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: write initial oid 393 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: seq_num 392 ranges {0=626688,626688=401408} 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: writing mira0435601-393 from 0 to 626688 tid 1 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: writing mira0435601-393 from 626688 to 1028096 tid 2 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 1 to mira0435601-377 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 2 to mira0435601-377 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 3 to mira0435601-377 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 4 to mira0435601-377 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 6 to mira0435601-377 2014-07-01 16:17:25,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 7 to mira0435601-376 2014-07-01 16:17:25,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 81 (ObjNum 375 snap 0 seq_num 375) dirty exists 2014-07-01 16:17:25,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 5 to mira0435601-378 2014-07-01 16:17:25,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 378 v 72 (ObjNum 377 snap 0 seq_num 377) dirty exists 2014-07-01 16:17:25,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 7 to mira0435601-377 2014-07-01 16:17:25,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 76 (ObjNum 376 snap 0 seq_num 376) dirty exists 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 1 to mira0435601-380 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 2 to mira0435601-380 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 3 to mira0435601-380 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 1 to mira0435601-382 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 3 to mira0435601-382 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 1 to mira0435601-383 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 2 to mira0435601-383 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 3 to mira0435601-383 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 5 to mira0435601-380 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 4 to mira0435601-383 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 6 to mira0435601-383 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 1 to mira0435601-384 2014-07-01 16:17:25,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 2 to mira0435601-384 2014-07-01 16:17:25,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 3 to mira0435601-384 2014-07-01 16:17:25,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 5 to mira0435601-384 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 4 to mira0435601-382 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 52 (ObjNum 381 snap 0 seq_num 381) dirty exists 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 7 to mira0435601-383 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 57 (ObjNum 382 snap 0 seq_num 382) dirty exists 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 6 to mira0435601-380 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 380 v 85 (ObjNum 379 snap 0 seq_num 379) dirty exists 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 6 to mira0435601-384 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 74 (ObjNum 383 snap 0 seq_num 383) dirty exists 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: done (15 left) 2014-07-01 16:17:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: done (14 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: done (13 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: done (12 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: done (11 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: done (10 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: done (9 left) 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: write initial oid 394 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: seq_num 393 ranges {0=544768,544768=454656,999424=626688,1626112=122880} 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: writing mira0435601-394 from 0 to 544768 tid 1 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: writing mira0435601-394 from 544768 to 999424 tid 2 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: writing mira0435601-394 from 999424 to 1626112 tid 3 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: writing mira0435601-394 from 1626112 to 1748992 tid 4 2014-07-01 16:17:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: write initial oid 395 2014-07-01 16:17:25,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: seq_num 394 ranges {0=573440} 2014-07-01 16:17:25,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: writing mira0435601-395 from 0 to 573440 tid 1 2014-07-01 16:17:25,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: write initial oid 396 2014-07-01 16:17:25,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: seq_num 395 ranges {0=446464} 2014-07-01 16:17:25,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: writing mira0435601-396 from 0 to 446464 tid 1 2014-07-01 16:17:25,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: write initial oid 397 2014-07-01 16:17:25,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: seq_num 396 ranges {0=315392} 2014-07-01 16:17:25,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: writing mira0435601-397 from 0 to 315392 tid 1 2014-07-01 16:17:25,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: write initial oid 398 2014-07-01 16:17:25,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: seq_num 397 ranges {0=794624,794624=679936,1474560=520192,1994752=405504} 2014-07-01 16:17:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0435601-398 from 0 to 794624 tid 1 2014-07-01 16:17:26,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0435601-398 from 794624 to 1474560 tid 2 2014-07-01 16:17:26,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0435601-398 from 1474560 to 1994752 tid 3 2014-07-01 16:17:26,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0435601-398 from 1994752 to 2400256 tid 4 2014-07-01 16:17:26,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: write initial oid 399 2014-07-01 16:17:26,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: seq_num 398 ranges {0=667648,667648=667648,1335296=745472,2080768=192512} 2014-07-01 16:17:26,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0435601-399 from 0 to 667648 tid 1 2014-07-01 16:17:26,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0435601-399 from 667648 to 1335296 tid 2 2014-07-01 16:17:26,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0435601-399 from 1335296 to 2080768 tid 3 2014-07-01 16:17:26,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0435601-399 from 2080768 to 2273280 tid 4 2014-07-01 16:17:26,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: write initial oid 400 2014-07-01 16:17:26,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: seq_num 399 ranges {0=536576,536576=495616,1032192=417792,1449984=593920,2043904=98304} 2014-07-01 16:17:26,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0435601-400 from 0 to 536576 tid 1 2014-07-01 16:17:26,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0435601-400 from 536576 to 1032192 tid 2 2014-07-01 16:17:26,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0435601-400 from 1032192 to 1449984 tid 3 2014-07-01 16:17:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0435601-400 from 1449984 to 2043904 tid 4 2014-07-01 16:17:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0435601-400 from 2043904 to 2142208 tid 5 2014-07-01 16:17:26,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 1 to mira0435601-385 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 2 to mira0435601-385 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 3 to mira0435601-385 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 4 to mira0435601-385 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 6 to mira0435601-385 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 1 to mira0435601-387 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 3 to mira0435601-387 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 1 to mira0435601-389 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 2 to mira0435601-389 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 3 to mira0435601-389 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 5 to mira0435601-389 2014-07-01 16:17:26,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 1 to mira0435601-386 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 2 to mira0435601-386 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 3 to mira0435601-386 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 4 to mira0435601-386 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 6 to mira0435601-386 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 1 to mira0435601-390 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 3 to mira0435601-390 2014-07-01 16:17:26,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 1 to mira0435601-388 2014-07-01 16:17:26,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 3 to mira0435601-388 2014-07-01 16:17:26,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 1 to mira0435601-391 2014-07-01 16:17:26,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 2 to mira0435601-391 2014-07-01 16:17:26,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 3 to mira0435601-391 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 4 to mira0435601-391 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 1 to mira0435601-392 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 4 to mira0435601-387 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 64 (ObjNum 386 snap 0 seq_num 386) dirty exists 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 7 to mira0435601-385 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 385 v 70 (ObjNum 384 snap 0 seq_num 384) dirty exists 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 6 to mira0435601-389 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 65 (ObjNum 388 snap 0 seq_num 388) dirty exists 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: done (15 left) 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: done (14 left) 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: done (13 left) 2014-07-01 16:17:26,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: write initial oid 401 2014-07-01 16:17:26,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: seq_num 400 ranges {0=565248,565248=401408} 2014-07-01 16:17:26,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: writing mira0435601-401 from 0 to 565248 tid 1 2014-07-01 16:17:26,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: writing mira0435601-401 from 565248 to 966656 tid 2 2014-07-01 16:17:26,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: write initial oid 402 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: seq_num 401 ranges {0=434176,434176=401408} 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: writing mira0435601-402 from 0 to 434176 tid 1 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: writing mira0435601-402 from 434176 to 835584 tid 2 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: write initial oid 403 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: seq_num 402 ranges {0=757760,757760=647168,1404928=155648} 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: writing mira0435601-403 from 0 to 757760 tid 1 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: writing mira0435601-403 from 757760 to 1404928 tid 2 2014-07-01 16:17:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: writing mira0435601-403 from 1404928 to 1560576 tid 3 2014-07-01 16:17:26,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:26,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 4 to mira0435601-388 2014-07-01 16:17:26,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 106 (ObjNum 387 snap 0 seq_num 387) dirty exists 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 4 to mira0435601-390 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 390 v 72 (ObjNum 389 snap 0 seq_num 389) dirty exists 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 7 to mira0435601-386 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 97 (ObjNum 385 snap 0 seq_num 385) dirty exists 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 5 to mira0435601-391 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 7 to mira0435601-391 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 1 to mira0435601-395 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 3 to mira0435601-395 2014-07-01 16:17:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 1 to mira0435601-397 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 3 to mira0435601-397 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 8 to mira0435601-391 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 87 (ObjNum 390 snap 0 seq_num 390) dirty exists 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 4 to mira0435601-395 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 78 (ObjNum 394 snap 0 seq_num 394) dirty exists 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 4 to mira0435601-397 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 57 (ObjNum 396 snap 0 seq_num 396) dirty exists 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: done (15 left) 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: done (14 left) 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: done (13 left) 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: done (12 left) 2014-07-01 16:17:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: done (11 left) 2014-07-01 16:17:26,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: done (10 left) 2014-07-01 16:17:26,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: write initial oid 404 2014-07-01 16:17:26,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: seq_num 403 ranges {0=385024} 2014-07-01 16:17:26,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: writing mira0435601-404 from 0 to 385024 tid 1 2014-07-01 16:17:26,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: write initial oid 405 2014-07-01 16:17:26,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: seq_num 404 ranges {0=409600,409600=462848,872448=536576,1409024=204800} 2014-07-01 16:17:26,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: writing mira0435601-405 from 0 to 409600 tid 1 2014-07-01 16:17:26,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: writing mira0435601-405 from 409600 to 872448 tid 2 2014-07-01 16:17:26,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: writing mira0435601-405 from 872448 to 1409024 tid 3 2014-07-01 16:17:26,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: writing mira0435601-405 from 1409024 to 1613824 tid 4 2014-07-01 16:17:26,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: write initial oid 406 2014-07-01 16:17:26,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: seq_num 405 ranges {0=126976} 2014-07-01 16:17:26,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: writing mira0435601-406 from 0 to 126976 tid 1 2014-07-01 16:17:26,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: write initial oid 407 2014-07-01 16:17:26,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: seq_num 406 ranges {0=606208,606208=471040,1077248=483328,1560576=651264} 2014-07-01 16:17:26,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:17:26.191471min lat: 0.067553 max lat: 14.2279 avg lat: 2.59745 2014-07-01 16:17:26,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:17:26,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 80 16 505 489 24.4447 88 0.433055 2.59745 2014-07-01 16:17:26,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0435601-407 from 0 to 606208 tid 1 2014-07-01 16:17:27,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0435601-407 from 606208 to 1077248 tid 2 2014-07-01 16:17:27,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0435601-407 from 1077248 to 1560576 tid 3 2014-07-01 16:17:27,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0435601-407 from 1560576 to 2211840 tid 4 2014-07-01 16:17:27,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: write initial oid 408 2014-07-01 16:17:27,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: seq_num 407 ranges {0=630784,630784=401408} 2014-07-01 16:17:27,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: writing mira0435601-408 from 0 to 630784 tid 1 2014-07-01 16:17:27,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: writing mira0435601-408 from 630784 to 1032192 tid 2 2014-07-01 16:17:27,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: write initial oid 409 2014-07-01 16:17:27,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: seq_num 408 ranges {0=749568,749568=688128,1437696=516096} 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: writing mira0435601-409 from 0 to 749568 tid 1 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: writing mira0435601-409 from 749568 to 1437696 tid 2 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: writing mira0435601-409 from 1437696 to 1953792 tid 3 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 2 to mira0435601-392 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 4 to mira0435601-392 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 1 to mira0435601-398 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 1 to mira0435601-393 2014-07-01 16:17:27,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 2 to mira0435601-393 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 4 to mira0435601-393 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 1 to mira0435601-394 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 2 to mira0435601-398 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 2 to mira0435601-394 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 3 to mira0435601-394 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 1 to mira0435601-396 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 3 to mira0435601-396 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 4 to mira0435601-394 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 3 to mira0435601-398 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 6 to mira0435601-394 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 5 to mira0435601-392 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 392 v 102 (ObjNum 391 snap 0 seq_num 391) dirty exists 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: done (15 left) 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: write initial oid 410 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: seq_num 409 ranges {0=778240} 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: writing mira0435601-410 from 0 to 778240 tid 1 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 4 to mira0435601-396 2014-07-01 16:17:27,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 99 (ObjNum 395 snap 0 seq_num 395) dirty exists 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 4 to mira0435601-398 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 6 to mira0435601-398 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 5 to mira0435601-393 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 393 v 65 (ObjNum 392 snap 0 seq_num 392) dirty exists 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 7 to mira0435601-394 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 71 (ObjNum 393 snap 0 seq_num 393) dirty exists 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 7 to mira0435601-398 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 61 (ObjNum 397 snap 0 seq_num 397) dirty exists 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: done (15 left) 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: done (14 left) 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: done (13 left) 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: done (12 left) 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: write initial oid 411 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: seq_num 410 ranges {0=802816,802816=454656,1257472=532480,1789952=217088} 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0435601-411 from 0 to 802816 tid 1 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0435601-411 from 802816 to 1257472 tid 2 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0435601-411 from 1257472 to 1789952 tid 3 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0435601-411 from 1789952 to 2007040 tid 4 2014-07-01 16:17:27,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: write initial oid 412 2014-07-01 16:17:27,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: seq_num 411 ranges {0=569344,569344=684032,1253376=118784} 2014-07-01 16:17:27,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: writing mira0435601-412 from 0 to 569344 tid 1 2014-07-01 16:17:27,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: writing mira0435601-412 from 569344 to 1253376 tid 2 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: writing mira0435601-412 from 1253376 to 1372160 tid 3 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: write initial oid 413 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: seq_num 412 ranges {0=196608} 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: writing mira0435601-413 from 0 to 196608 tid 1 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: write initial oid 414 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: seq_num 413 ranges {0=622592,622592=450560,1073152=352256} 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: writing mira0435601-414 from 0 to 622592 tid 1 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: writing mira0435601-414 from 622592 to 1073152 tid 2 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: writing mira0435601-414 from 1073152 to 1425408 tid 3 2014-07-01 16:17:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:27,896.896 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 81 16 513 497 24.5379 32 0.297699 2.5714 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 1 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 2 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 3 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 4 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 6 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 1 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 2 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 3 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 4 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 5 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 7 to mira0435601-400 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 7 to mira0435601-399 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 399 v 78 (ObjNum 398 snap 0 seq_num 398) dirty exists 2014-07-01 16:17:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: done (15 left) 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: write initial oid 415 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: seq_num 414 ranges {0=741376,741376=434176,1175552=532480,1708032=512000,2220032=126976} 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0435601-415 from 0 to 741376 tid 1 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0435601-415 from 741376 to 1175552 tid 2 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0435601-415 from 1175552 to 1708032 tid 3 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0435601-415 from 1708032 to 2220032 tid 4 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0435601-415 from 2220032 to 2347008 tid 5 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 1 to mira0435601-402 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 2 to mira0435601-402 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 4 to mira0435601-402 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 1 to mira0435601-401 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 2 to mira0435601-401 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 4 to mira0435601-401 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 5 to mira0435601-401 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 60 (ObjNum 400 snap 0 seq_num 400) dirty exists 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 5 to mira0435601-402 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 74 (ObjNum 401 snap 0 seq_num 401) dirty exists 2014-07-01 16:17:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: done (15 left) 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: done (14 left) 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: write initial oid 416 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: seq_num 415 ranges {0=417792,417792=663552,1081344=405504,1486848=536576} 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0435601-416 from 0 to 417792 tid 1 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0435601-416 from 417792 to 1081344 tid 2 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0435601-416 from 1081344 to 1486848 tid 3 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0435601-416 from 1486848 to 2023424 tid 4 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: write initial oid 417 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: seq_num 416 ranges {0=442368,442368=401408} 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: writing mira0435601-417 from 0 to 442368 tid 1 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: writing mira0435601-417 from 442368 to 843776 tid 2 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 1 to mira0435601-403 2014-07-01 16:17:27,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 2 to mira0435601-403 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 3 to mira0435601-403 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 5 to mira0435601-403 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 6 to mira0435601-403 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 111 (ObjNum 402 snap 0 seq_num 402) dirty exists 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: done (15 left) 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: write initial oid 418 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: seq_num 417 ranges {0=716800} 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: writing mira0435601-418 from 0 to 716800 tid 1 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:28,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 1 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 2 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 3 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 4 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 6 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 1 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 2 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 3 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 4 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 1 to mira0435601-408 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 6 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 2 to mira0435601-408 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 4 to mira0435601-408 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 7 to mira0435601-405 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 79 (ObjNum 404 snap 0 seq_num 404) dirty exists 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: done (15 left) 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: write initial oid 419 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: seq_num 418 ranges {0=589824} 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: writing mira0435601-419 from 0 to 589824 tid 1 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 7 to mira0435601-407 2014-07-01 16:17:28,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 407 v 84 (ObjNum 406 snap 0 seq_num 406) dirty exists 2014-07-01 16:17:28,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 5 to mira0435601-408 2014-07-01 16:17:28,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 85 (ObjNum 407 snap 0 seq_num 407) dirty exists 2014-07-01 16:17:28,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 8 to mira0435601-400 2014-07-01 16:17:28,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 82 (ObjNum 399 snap 0 seq_num 399) dirty exists 2014-07-01 16:17:28,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: done (15 left) 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: done (14 left) 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: done (13 left) 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: write initial oid 420 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: seq_num 419 ranges {0=458752} 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: writing mira0435601-420 from 0 to 458752 tid 1 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: write initial oid 421 2014-07-01 16:17:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: seq_num 420 ranges {0=135168} 2014-07-01 16:17:29,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: writing mira0435601-421 from 0 to 135168 tid 1 2014-07-01 16:17:29,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: write initial oid 422 2014-07-01 16:17:29,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: seq_num 421 ranges {0=8192} 2014-07-01 16:17:29,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: writing mira0435601-422 from 0 to 8192 tid 1 2014-07-01 16:17:29,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 1 to mira0435601-410 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 3 to mira0435601-410 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 4 to mira0435601-410 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 113 (ObjNum 409 snap 0 seq_num 409) dirty exists 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: done (15 left) 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: write initial oid 423 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: seq_num 422 ranges {0=679936,679936=638976,1318912=471040,1789952=421888,2211840=73728} 2014-07-01 16:17:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0435601-423 from 0 to 679936 tid 1 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0435601-423 from 679936 to 1318912 tid 2 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0435601-423 from 1318912 to 1789952 tid 3 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0435601-423 from 1789952 to 2211840 tid 4 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0435601-423 from 2211840 to 2285568 tid 5 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 1 to mira0435601-414 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 2 to mira0435601-414 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 1 to mira0435601-413 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 3 to mira0435601-413 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 4 to mira0435601-413 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 87 (ObjNum 412 snap 0 seq_num 412) dirty exists 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: done (15 left) 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: write initial oid 424 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: seq_num 423 ranges {0=708608,708608=401408} 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: writing mira0435601-424 from 0 to 708608 tid 1 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: writing mira0435601-424 from 708608 to 1110016 tid 2 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 1 to mira0435601-406 2014-07-01 16:17:29,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 3 to mira0435601-406 2014-07-01 16:17:29,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 1 to mira0435601-404 2014-07-01 16:17:29,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 3 to mira0435601-404 2014-07-01 16:17:29,179.179 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 82 16 518 502 24.4825 20 2.11007 2.56557 2014-07-01 16:17:29,279.279 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 83 16 528 512 24.6693 40 3.40166 2.55994 2014-07-01 16:17:30,278.278 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 84 16 529 513 24.4232 4 0.470196 2.55587 2014-07-01 16:17:30,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 1 to mira0435601-409 2014-07-01 16:17:30,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 2 to mira0435601-409 2014-07-01 16:17:30,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 3 to mira0435601-409 2014-07-01 16:17:30,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 5 to mira0435601-409 2014-07-01 16:17:30,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 1 to mira0435601-411 2014-07-01 16:17:30,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 2 to mira0435601-411 2014-07-01 16:17:30,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 3 to mira0435601-411 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 1 to mira0435601-412 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 4 to mira0435601-404 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 108 (ObjNum 403 snap 0 seq_num 403) dirty exists 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: done (15 left) 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: write initial oid 425 2014-07-01 16:17:30,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: seq_num 424 ranges {0=626688,626688=454656,1081344=524288,1605632=225280} 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0435601-425 from 0 to 626688 tid 1 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0435601-425 from 626688 to 1081344 tid 2 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0435601-425 from 1081344 to 1605632 tid 3 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0435601-425 from 1605632 to 1830912 tid 4 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 4 to mira0435601-406 2014-07-01 16:17:30,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 406 v 86 (ObjNum 405 snap 0 seq_num 405) dirty exists 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 6 to mira0435601-409 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 103 (ObjNum 408 snap 0 seq_num 408) dirty exists 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: done (15 left) 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: done (14 left) 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: write initial oid 426 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: seq_num 425 ranges {0=499712,499712=634880,1134592=569344} 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: writing mira0435601-426 from 0 to 499712 tid 1 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: writing mira0435601-426 from 499712 to 1134592 tid 2 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: writing mira0435601-426 from 1134592 to 1703936 tid 3 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: write initial oid 427 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: seq_num 426 ranges {0=528384} 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: writing mira0435601-427 from 0 to 528384 tid 1 2014-07-01 16:17:30,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:30,833.833 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:30,833.833 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:30,833.833 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 3, 4, 5, 2] 2014-07-01 16:17:30,833.833 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 16:17:31,279.279 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 85 16 532 516 24.277 12 1.59152 2.56042 2014-07-01 16:17:31,418.418 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 16:17:31,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 1 to mira0435601-416 2014-07-01 16:17:31,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 2 to mira0435601-416 2014-07-01 16:17:31,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 3 to mira0435601-416 2014-07-01 16:17:31,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 4 to mira0435601-416 2014-07-01 16:17:31,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 6 to mira0435601-416 2014-07-01 16:17:31,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 7 to mira0435601-416 2014-07-01 16:17:31,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 92 (ObjNum 415 snap 0 seq_num 415) dirty exists 2014-07-01 16:17:31,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: done (15 left) 2014-07-01 16:17:31,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: write initial oid 428 2014-07-01 16:17:31,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: seq_num 427 ranges {0=397312} 2014-07-01 16:17:31,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: writing mira0435601-428 from 0 to 397312 tid 1 2014-07-01 16:17:31,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 1 to mira0435601-422 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 3 to mira0435601-422 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 4 to mira0435601-422 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 88 (ObjNum 421 snap 0 seq_num 421) dirty exists 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: done (15 left) 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: write initial oid 429 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: seq_num 428 ranges {0=270336} 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: writing mira0435601-429 from 0 to 270336 tid 1 2014-07-01 16:17:31,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:32,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 1 to mira0435601-420 2014-07-01 16:17:32,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 3 to mira0435601-420 2014-07-01 16:17:32,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 2 to mira0435601-412 2014-07-01 16:17:32,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 4 to mira0435601-420 2014-07-01 16:17:32,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 105 (ObjNum 419 snap 0 seq_num 419) dirty exists 2014-07-01 16:17:32,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: done (15 left) 2014-07-01 16:17:32,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: write initial oid 430 2014-07-01 16:17:32,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: seq_num 429 ranges {0=749568,749568=462848,1212416=491520,1703936=651264} 2014-07-01 16:17:32,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0435601-430 from 0 to 749568 tid 1 2014-07-01 16:17:32,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0435601-430 from 749568 to 1212416 tid 2 2014-07-01 16:17:32,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0435601-430 from 1212416 to 1703936 tid 3 2014-07-01 16:17:32,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0435601-430 from 1703936 to 2355200 tid 4 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 3 to mira0435601-412 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 5 to mira0435601-412 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 1 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 2 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 3 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 4 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 5 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 7 to mira0435601-423 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 1 to mira0435601-424 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 2 to mira0435601-424 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 4 to mira0435601-424 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 1 to mira0435601-417 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 2 to mira0435601-417 2014-07-01 16:17:32,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 4 to mira0435601-417 2014-07-01 16:17:32,279.279 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 86 16 536 520 24.1808 16 3.42238 2.57089 2014-07-01 16:17:32,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 5 to mira0435601-424 2014-07-01 16:17:32,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 85 (ObjNum 423 snap 0 seq_num 423) dirty exists 2014-07-01 16:17:32,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: done (15 left) 2014-07-01 16:17:32,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: write initial oid 431 2014-07-01 16:17:32,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: seq_num 430 ranges {0=774144,774144=401408} 2014-07-01 16:17:32,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: writing mira0435601-431 from 0 to 774144 tid 1 2014-07-01 16:17:32,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: writing mira0435601-431 from 774144 to 1175552 tid 2 2014-07-01 16:17:32,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:32,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 1 to mira0435601-430 2014-07-01 16:17:32,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 2 to mira0435601-430 2014-07-01 16:17:32,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 3 to mira0435601-430 2014-07-01 16:17:33,279.279 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 87 16 539 523 24.0407 12 3.51163 2.57609 2014-07-01 16:17:34,280.280 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 88 16 539 523 23.7675 0 - 2.57609 2014-07-01 16:17:34,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 1 to mira0435601-425 2014-07-01 16:17:34,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 2 to mira0435601-425 2014-07-01 16:17:34,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 3 to mira0435601-425 2014-07-01 16:17:34,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 4 to mira0435601-425 2014-07-01 16:17:34,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 6 to mira0435601-425 2014-07-01 16:17:34,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 1 to mira0435601-426 2014-07-01 16:17:34,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 2 to mira0435601-426 2014-07-01 16:17:34,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 3 to mira0435601-426 2014-07-01 16:17:34,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 5 to mira0435601-426 2014-07-01 16:17:34,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 1 to mira0435601-427 2014-07-01 16:17:34,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 3 to mira0435601-427 2014-07-01 16:17:34,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 4 to mira0435601-430 2014-07-01 16:17:34,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 5 to mira0435601-417 2014-07-01 16:17:34,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 75 (ObjNum 416 snap 0 seq_num 416) dirty exists 2014-07-01 16:17:34,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: done (15 left) 2014-07-01 16:17:34,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: write initial oid 432 2014-07-01 16:17:34,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: seq_num 431 ranges {0=491520,491520=675840,1167360=438272,1605632=491520} 2014-07-01 16:17:34,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0435601-432 from 0 to 491520 tid 1 2014-07-01 16:17:34,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0435601-432 from 491520 to 1167360 tid 2 2014-07-01 16:17:34,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0435601-432 from 1167360 to 1605632 tid 3 2014-07-01 16:17:34,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0435601-432 from 1605632 to 2097152 tid 4 2014-07-01 16:17:34,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:34,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 8 to mira0435601-423 2014-07-01 16:17:34,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 104 (ObjNum 422 snap 0 seq_num 422) dirty exists 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 6 to mira0435601-412 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 412 v 82 (ObjNum 411 snap 0 seq_num 411) dirty exists 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: done (15 left) 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: done (14 left) 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: write initial oid 433 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: seq_num 432 ranges {0=520192,520192=401408} 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: writing mira0435601-433 from 0 to 520192 tid 1 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: writing mira0435601-433 from 520192 to 921600 tid 2 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: write initial oid 434 2014-07-01 16:17:34,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: seq_num 433 ranges {0=593920} 2014-07-01 16:17:34,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: writing mira0435601-434 from 0 to 593920 tid 1 2014-07-01 16:17:34,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:35,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 6 to mira0435601-430 2014-07-01 16:17:35,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 7 to mira0435601-430 2014-07-01 16:17:35,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 99 (ObjNum 429 snap 0 seq_num 429) dirty exists 2014-07-01 16:17:35,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: done (15 left) 2014-07-01 16:17:35,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: write initial oid 435 2014-07-01 16:17:35,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: seq_num 434 ranges {0=712704,712704=425984,1138688=376832} 2014-07-01 16:17:35,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: writing mira0435601-435 from 0 to 712704 tid 1 2014-07-01 16:17:35,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: writing mira0435601-435 from 712704 to 1138688 tid 2 2014-07-01 16:17:35,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: writing mira0435601-435 from 1138688 to 1515520 tid 3 2014-07-01 16:17:35,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:35,280.280 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 89 16 542 526 23.6353 6 2.7653 2.5793 2014-07-01 16:17:35,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 7 to mira0435601-425 2014-07-01 16:17:35,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 64 (ObjNum 424 snap 0 seq_num 424) dirty exists 2014-07-01 16:17:35,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: done (15 left) 2014-07-01 16:17:35,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: write initial oid 436 2014-07-01 16:17:35,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: seq_num 435 ranges {0=339968} 2014-07-01 16:17:35,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: writing mira0435601-436 from 0 to 339968 tid 1 2014-07-01 16:17:35,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:35,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 6 to mira0435601-426 2014-07-01 16:17:35,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 426 v 69 (ObjNum 425 snap 0 seq_num 425) dirty exists 2014-07-01 16:17:35,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: done (15 left) 2014-07-01 16:17:35,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: write initial oid 437 2014-07-01 16:17:35,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: seq_num 436 ranges {0=765952,765952=643072,1409024=159744} 2014-07-01 16:17:35,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: writing mira0435601-437 from 0 to 765952 tid 1 2014-07-01 16:17:35,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: writing mira0435601-437 from 765952 to 1409024 tid 2 2014-07-01 16:17:35,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: writing mira0435601-437 from 1409024 to 1568768 tid 3 2014-07-01 16:17:35,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:36,280.280 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 90 16 545 529 23.506 12 7.33631 2.59184 2014-07-01 16:17:36,434.434 INFO:teuthology.task.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 2] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:36,434.434 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:36,434.434 INFO:teuthology.task.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 3, 4, 5, 2] 2014-07-01 16:17:36,434.434 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 0' 2014-07-01 16:17:37,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 4 to mira0435601-427 2014-07-01 16:17:37,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 66 (ObjNum 426 snap 0 seq_num 426) dirty exists 2014-07-01 16:17:37,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: done (15 left) 2014-07-01 16:17:37,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: write initial oid 438 2014-07-01 16:17:37,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: seq_num 437 ranges {0=81920} 2014-07-01 16:17:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: writing mira0435601-438 from 0 to 81920 tid 1 2014-07-01 16:17:37,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 1 to mira0435601-436 2014-07-01 16:17:37,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 3 to mira0435601-436 2014-07-01 16:17:37,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 1 to mira0435601-437 2014-07-01 16:17:37,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 2 to mira0435601-437 2014-07-01 16:17:37,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 1 to mira0435601-432 2014-07-01 16:17:37,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 3 to mira0435601-437 2014-07-01 16:17:37,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 5 to mira0435601-437 2014-07-01 16:17:37,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 91 16 548 532 23.3795 12 2.74813 2.59477 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 4 to mira0435601-436 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 90 (ObjNum 435 snap 0 seq_num 435) dirty exists 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: done (15 left) 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: write initial oid 439 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: seq_num 438 ranges {0=561152,561152=655360,1216512=659456,1875968=290816} 2014-07-01 16:17:37,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0435601-439 from 0 to 561152 tid 1 2014-07-01 16:17:37,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0435601-439 from 561152 to 1216512 tid 2 2014-07-01 16:17:37,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0435601-439 from 1216512 to 1875968 tid 3 2014-07-01 16:17:37,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0435601-439 from 1875968 to 2166784 tid 4 2014-07-01 16:17:37,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 6 to mira0435601-437 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 94 (ObjNum 436 snap 0 seq_num 436) dirty exists 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 4 to mira0435601-411 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 6 to mira0435601-411 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 7 to mira0435601-411 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 70 (ObjNum 410 snap 0 seq_num 410) dirty exists 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: done (15 left) 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: done (14 left) 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: write initial oid 440 2014-07-01 16:17:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: seq_num 439 ranges {0=585728,585728=401408} 2014-07-01 16:17:37,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: writing mira0435601-440 from 0 to 585728 tid 1 2014-07-01 16:17:37,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: writing mira0435601-440 from 585728 to 987136 tid 2 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: write initial oid 441 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: seq_num 440 ranges {0=458752,458752=401408} 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: writing mira0435601-441 from 0 to 458752 tid 1 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: writing mira0435601-441 from 458752 to 860160 tid 2 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 1 to mira0435601-421 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 3 to mira0435601-421 2014-07-01 16:17:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 4 to mira0435601-421 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 89 (ObjNum 420 snap 0 seq_num 420) dirty exists 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: done (15 left) 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: write initial oid 442 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: seq_num 441 ranges {0=733184} 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: writing mira0435601-442 from 0 to 733184 tid 1 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 1 to mira0435601-434 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 3 to mira0435601-434 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 4 to mira0435601-434 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 96 (ObjNum 433 snap 0 seq_num 433) dirty exists 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: done (15 left) 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: write initial oid 443 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: seq_num 442 ranges {0=405504} 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: writing mira0435601-443 from 0 to 405504 tid 1 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 1 to mira0435601-435 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 2 to mira0435601-432 2014-07-01 16:17:37,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 3 to mira0435601-432 2014-07-01 16:17:37,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 4 to mira0435601-432 2014-07-01 16:17:37,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 1 to mira0435601-431 2014-07-01 16:17:37,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 2 to mira0435601-431 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 4 to mira0435601-431 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 6 to mira0435601-432 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 7 to mira0435601-432 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 74 (ObjNum 431 snap 0 seq_num 431) dirty exists 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: done (15 left) 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: write initial oid 444 2014-07-01 16:17:37,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: seq_num 443 ranges {0=278528} 2014-07-01 16:17:37,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: writing mira0435601-444 from 0 to 278528 tid 1 2014-07-01 16:17:37,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 1 to mira0435601-428 2014-07-01 16:17:37,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 3 to mira0435601-428 2014-07-01 16:17:37,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 1 to mira0435601-429 2014-07-01 16:17:37,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 5 to mira0435601-431 2014-07-01 16:17:37,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 92 (ObjNum 430 snap 0 seq_num 430) dirty exists 2014-07-01 16:17:37,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 3 to mira0435601-429 2014-07-01 16:17:37,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: done (15 left) 2014-07-01 16:17:37,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: write initial oid 445 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: seq_num 444 ranges {0=147456} 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: writing mira0435601-445 from 0 to 147456 tid 1 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 1 to mira0435601-433 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 2 to mira0435601-433 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 4 to mira0435601-433 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 1 to mira0435601-438 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 3 to mira0435601-438 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 3 to mira0435601-414 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 5 to mira0435601-414 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 4 to mira0435601-429 2014-07-01 16:17:37,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 65 (ObjNum 428 snap 0 seq_num 428) dirty exists 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: done (15 left) 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: write initial oid 446 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: seq_num 445 ranges {0=577536,577536=786432,1363968=16384} 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: writing mira0435601-446 from 0 to 577536 tid 1 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: writing mira0435601-446 from 577536 to 1363968 tid 2 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: writing mira0435601-446 from 1363968 to 1380352 tid 3 2014-07-01 16:17:37,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 4 to mira0435601-428 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 63 (ObjNum 427 snap 0 seq_num 427) dirty exists 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 6 to mira0435601-414 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 117 (ObjNum 413 snap 0 seq_num 413) dirty exists 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: done (15 left) 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: done (14 left) 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: write initial oid 447 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: seq_num 446 ranges {0=450560,450560=614400,1064960=188416} 2014-07-01 16:17:37,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: writing mira0435601-447 from 0 to 450560 tid 1 2014-07-01 16:17:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: writing mira0435601-447 from 450560 to 1064960 tid 2 2014-07-01 16:17:37,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: writing mira0435601-447 from 1064960 to 1253376 tid 3 2014-07-01 16:17:37,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: write initial oid 448 2014-07-01 16:17:37,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: seq_num 447 ranges {0=770048,770048=598016,1368064=425984,1794048=180224} 2014-07-01 16:17:37,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0435601-448 from 0 to 770048 tid 1 2014-07-01 16:17:37,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0435601-448 from 770048 to 1368064 tid 2 2014-07-01 16:17:37,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0435601-448 from 1368064 to 1794048 tid 3 2014-07-01 16:17:37,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0435601-448 from 1794048 to 1974272 tid 4 2014-07-01 16:17:37,843.843 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.0. 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 1 to mira0435601-418 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 3 to mira0435601-418 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 4 to mira0435601-418 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 76 (ObjNum 417 snap 0 seq_num 417) dirty exists 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: done (15 left) 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: write initial oid 449 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: seq_num 448 ranges {0=798720} 2014-07-01 16:17:37,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: writing mira0435601-449 from 0 to 798720 tid 1 2014-07-01 16:17:37,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:38,282.282 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 92 16 555 539 23.4296 28 0.918883 2.59338 2014-07-01 16:17:38,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 1 to mira0435601-440 2014-07-01 16:17:38,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 2 to mira0435601-440 2014-07-01 16:17:38,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 4 to mira0435601-440 2014-07-01 16:17:38,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 1 to mira0435601-441 2014-07-01 16:17:38,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 2 to mira0435601-441 2014-07-01 16:17:39,281.281 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 93 16 560 544 23.3927 20 1.49577 2.58775 2014-07-01 16:17:39,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 4 to mira0435601-441 2014-07-01 16:17:39,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 5 to mira0435601-440 2014-07-01 16:17:39,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 60 (ObjNum 439 snap 0 seq_num 439) dirty exists 2014-07-01 16:17:39,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: done (15 left) 2014-07-01 16:17:39,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: write initial oid 450 2014-07-01 16:17:39,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: seq_num 449 ranges {0=671744} 2014-07-01 16:17:39,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: writing mira0435601-450 from 0 to 671744 tid 1 2014-07-01 16:17:39,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:40,281.281 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 94 16 562 546 23.2289 8 1.96338 2.58397 2014-07-01 16:17:41,282.282 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 95 16 564 548 23.0686 8 1.05937 2.57925 2014-07-01 16:17:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 5 to mira0435601-441 2014-07-01 16:17:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 88 (ObjNum 440 snap 0 seq_num 440) dirty exists 2014-07-01 16:17:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: done (15 left) 2014-07-01 16:17:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: write initial oid 451 2014-07-01 16:17:41,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: seq_num 450 ranges {0=540672} 2014-07-01 16:17:41,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: writing mira0435601-451 from 0 to 540672 tid 1 2014-07-01 16:17:41,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:42,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 1 to mira0435601-450 2014-07-01 16:17:42,281.281 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 96 16 568 552 22.9949 16 1.76275 2.57906 2014-07-01 16:17:42,856.856 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2] out_osds: [1, 0] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:42,857.857 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:42,857.857 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 16:17:42,857.857 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 16:17:43,282.282 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 97 16 568 552 22.7579 0 - 2.57906 2014-07-01 16:17:44,283.283 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 98 16 568 552 22.5256 0 - 2.57906 2014-07-01 16:17:45,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 99 16 568 552 22.2981 0 - 2.57906 2014-07-01 16:17:46,283.283 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:17:46.196184min lat: 0.067553 max lat: 14.2279 avg lat: 2.57906 2014-07-01 16:17:46,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:17:46,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 100 16 568 552 22.0751 0 - 2.57906 2014-07-01 16:17:47,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 101 16 573 557 22.0545 4 5.29691 2.61174 2014-07-01 16:17:47,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 3 to mira0435601-450 2014-07-01 16:17:47,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 1 to mira0435601-449 2014-07-01 16:17:47,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 3 to mira0435601-449 2014-07-01 16:17:47,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 1 to mira0435601-448 2014-07-01 16:17:47,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 2 to mira0435601-448 2014-07-01 16:17:47,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 3 to mira0435601-448 2014-07-01 16:17:47,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 4 to mira0435601-448 2014-07-01 16:17:47,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 6 to mira0435601-448 2014-07-01 16:17:47,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 1 to mira0435601-419 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 1 to mira0435601-442 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 1 to mira0435601-443 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 3 to mira0435601-443 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 4 to mira0435601-449 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 104 (ObjNum 448 snap 0 seq_num 448) dirty exists 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: done (15 left) 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: write initial oid 452 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: seq_num 451 ranges {0=217088} 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: writing mira0435601-452 from 0 to 217088 tid 1 2014-07-01 16:17:47,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:48,018.018 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 16:17:48,027.027 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 16:17:48,283.283 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 102 16 574 558 21.8775 4 0.263556 2.60753 2014-07-01 16:17:48,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 3 to mira0435601-442 2014-07-01 16:17:48,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 4 to mira0435601-450 2014-07-01 16:17:48,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 450 v 90 (ObjNum 449 snap 0 seq_num 449) dirty exists 2014-07-01 16:17:48,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: done (15 left) 2014-07-01 16:17:48,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: write initial oid 453 2014-07-01 16:17:48,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: seq_num 452 ranges {0=90112} 2014-07-01 16:17:48,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: writing mira0435601-453 from 0 to 90112 tid 1 2014-07-01 16:17:48,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:48,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 1 to mira0435601-453 2014-07-01 16:17:48,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 3 to mira0435601-453 2014-07-01 16:17:48,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 4 to mira0435601-453 2014-07-01 16:17:48,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 94 (ObjNum 452 snap 0 seq_num 452) dirty exists 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: done (15 left) 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: write initial oid 454 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: seq_num 453 ranges {0=516096,516096=589824,1105920=212992} 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: writing mira0435601-454 from 0 to 516096 tid 1 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: writing mira0435601-454 from 516096 to 1105920 tid 2 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: writing mira0435601-454 from 1105920 to 1318912 tid 3 2014-07-01 16:17:48,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:49,283.283 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 103 16 577 561 21.7816 12 0.099567 2.59927 2014-07-01 16:17:50,285.285 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 104 16 579 563 21.649 8 21.191 2.66924 2014-07-01 16:17:51,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 7 to mira0435601-448 2014-07-01 16:17:51,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 104 (ObjNum 447 snap 0 seq_num 447) dirty exists 2014-07-01 16:17:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: done (15 left) 2014-07-01 16:17:51,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: write initial oid 455 2014-07-01 16:17:51,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: seq_num 454 ranges {0=790528,790528=401408} 2014-07-01 16:17:51,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: writing mira0435601-455 from 0 to 790528 tid 1 2014-07-01 16:17:51,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: writing mira0435601-455 from 790528 to 1191936 tid 2 2014-07-01 16:17:51,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:51,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 105 16 587 571 21.7476 32 1.13869 2.81632 2014-07-01 16:17:52,284.284 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 106 16 591 575 21.6933 16 0.686894 2.79959 2014-07-01 16:17:53,028.028 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1] out_osds: [0] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:17:53,028.028 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:17:53,029.029 INFO:teuthology.task.thrashosds.thrasher:Adding osd 0 2014-07-01 16:17:53,029.029 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 0' 2014-07-01 16:17:53,283.283 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 107 16 591 575 21.4906 0 - 2.79959 2014-07-01 16:17:54,285.285 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 108 16 591 575 21.2916 0 - 2.79959 2014-07-01 16:17:55,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 109 16 591 575 21.0963 0 - 2.79959 2014-07-01 16:17:56,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 3 to mira0435601-419 2014-07-01 16:17:56,285.285 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 110 16 595 579 21.0499 4 25.6695 2.85805 2014-07-01 16:17:57,217.217 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.0. 2014-07-01 16:17:57,227.227 INFO:teuthology.task.thrashosds.thrasher:Added osd 0 2014-07-01 16:17:57,291.291 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 111 16 600 584 21.0404 20 5.59726 2.87629 2014-07-01 16:17:58,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 112 16 600 584 20.8525 0 - 2.87629 2014-07-01 16:17:59,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 113 16 600 584 20.668 0 - 2.87629 2014-07-01 16:17:59,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 1 to mira0435601-444 2014-07-01 16:17:59,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 3 to mira0435601-444 2014-07-01 16:17:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 5 to mira0435601-433 2014-07-01 16:17:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 88 (ObjNum 432 snap 0 seq_num 432) dirty exists 2014-07-01 16:17:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: done (15 left) 2014-07-01 16:17:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: write initial oid 456 2014-07-01 16:17:59,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: seq_num 455 ranges {0=663552,663552=401408} 2014-07-01 16:17:59,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: writing mira0435601-456 from 0 to 663552 tid 1 2014-07-01 16:17:59,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: writing mira0435601-456 from 663552 to 1064960 tid 2 2014-07-01 16:17:59,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 4 to mira0435601-438 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 90 (ObjNum 437 snap 0 seq_num 437) dirty exists 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 1 to mira0435601-451 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 3 to mira0435601-451 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 4 to mira0435601-451 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 78 (ObjNum 450 snap 0 seq_num 450) dirty exists 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: done (15 left) 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: done (14 left) 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: write initial oid 457 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: seq_num 456 ranges {0=737280} 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: writing mira0435601-457 from 0 to 737280 tid 1 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: write initial oid 458 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: seq_num 457 ranges {0=610304} 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: writing mira0435601-458 from 0 to 610304 tid 1 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 1 to mira0435601-415 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 2 to mira0435601-415 2014-07-01 16:17:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 3 to mira0435601-415 2014-07-01 16:17:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 4 to mira0435601-415 2014-07-01 16:17:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 5 to mira0435601-415 2014-07-01 16:17:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 7 to mira0435601-415 2014-07-01 16:17:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 1 to mira0435601-458 2014-07-01 16:17:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 3 to mira0435601-458 2014-07-01 16:17:59,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 4 to mira0435601-458 2014-07-01 16:17:59,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 110 (ObjNum 457 snap 0 seq_num 457) dirty exists 2014-07-01 16:17:59,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: done (15 left) 2014-07-01 16:17:59,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: write initial oid 459 2014-07-01 16:17:59,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: seq_num 458 ranges {0=483328} 2014-07-01 16:17:59,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: writing mira0435601-459 from 0 to 483328 tid 1 2014-07-01 16:17:59,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:00,285.285 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 114 16 608 592 20.7674 10.6667 0.12816 2.90801 2014-07-01 16:18:01,289.289 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 115 16 608 592 20.5868 0 - 2.90801 2014-07-01 16:18:02,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 2 to mira0435601-435 2014-07-01 16:18:02,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 3 to mira0435601-435 2014-07-01 16:18:02,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 5 to mira0435601-435 2014-07-01 16:18:02,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 6 to mira0435601-435 2014-07-01 16:18:02,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 121 (ObjNum 434 snap 0 seq_num 434) dirty exists 2014-07-01 16:18:02,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: done (15 left) 2014-07-01 16:18:02,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: write initial oid 460 2014-07-01 16:18:02,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: seq_num 459 ranges {0=507904,507904=630784,1138688=565248,1703936=8192} 2014-07-01 16:18:02,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0435601-460 from 0 to 507904 tid 1 2014-07-01 16:18:02,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0435601-460 from 507904 to 1138688 tid 2 2014-07-01 16:18:02,226.226 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:18:02,226.226 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:02,226.226 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [3, 4, 5, 2, 1, 0] 2014-07-01 16:18:02,227.227 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0435601-460 from 1138688 to 1703936 tid 3 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0435601-460 from 1703936 to 1712128 tid 4 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 1 to mira0435601-452 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 3 to mira0435601-452 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 1 to mira0435601-456 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 2 to mira0435601-456 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 1 to mira0435601-459 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 4 to mira0435601-456 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 3 to mira0435601-459 2014-07-01 16:18:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 8 to mira0435601-415 2014-07-01 16:18:02,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 415 v 90 (ObjNum 414 snap 0 seq_num 414) dirty exists 2014-07-01 16:18:02,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 4 to mira0435601-419 2014-07-01 16:18:02,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 92 (ObjNum 418 snap 0 seq_num 418) dirty exists 2014-07-01 16:18:02,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 4 to mira0435601-443 2014-07-01 16:18:02,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 96 (ObjNum 442 snap 0 seq_num 442) dirty exists 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 4 to mira0435601-452 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 109 (ObjNum 451 snap 0 seq_num 451) dirty exists 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 5 to mira0435601-456 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 97 (ObjNum 455 snap 0 seq_num 455) dirty exists 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 4 to mira0435601-459 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 111 (ObjNum 458 snap 0 seq_num 458) dirty exists 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: done (15 left) 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: done (14 left) 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: done (13 left) 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: done (12 left) 2014-07-01 16:18:02,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: done (11 left) 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: done (10 left) 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: write initial oid 461 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: seq_num 460 ranges {0=28672} 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: writing mira0435601-461 from 0 to 28672 tid 1 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: write initial oid 462 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: seq_num 461 ranges {0=704512,704512=446464,1150976=667648,1818624=487424,2306048=4096} 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0435601-462 from 0 to 704512 tid 1 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0435601-462 from 704512 to 1150976 tid 2 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0435601-462 from 1150976 to 1818624 tid 3 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0435601-462 from 1818624 to 2306048 tid 4 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0435601-462 from 2306048 to 2310144 tid 5 2014-07-01 16:18:02,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: write initial oid 463 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: seq_num 462 ranges {0=729088,729088=401408} 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: writing mira0435601-463 from 0 to 729088 tid 1 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: writing mira0435601-463 from 729088 to 1130496 tid 2 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: write initial oid 464 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: seq_num 463 ranges {0=446464,446464=614400,1060864=614400,1675264=376832} 2014-07-01 16:18:02,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0435601-464 from 0 to 446464 tid 1 2014-07-01 16:18:02,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 116 16 610 594 20.4783 4 2.59508 2.90699 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0435601-464 from 446464 to 1060864 tid 2 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0435601-464 from 1060864 to 1675264 tid 3 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0435601-464 from 1675264 to 2052096 tid 4 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: write initial oid 465 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: seq_num 464 ranges {0=471040,471040=401408} 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: writing mira0435601-465 from 0 to 471040 tid 1 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: writing mira0435601-465 from 471040 to 872448 tid 2 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: write initial oid 466 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: seq_num 465 ranges {0=548864} 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: writing mira0435601-466 from 0 to 548864 tid 1 2014-07-01 16:18:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:03,286.286 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 117 16 610 594 20.3032 0 - 2.90699 2014-07-01 16:18:03,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 1 to mira0435601-454 2014-07-01 16:18:03,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 2 to mira0435601-454 2014-07-01 16:18:03,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 3 to mira0435601-454 2014-07-01 16:18:03,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 5 to mira0435601-454 2014-07-01 16:18:03,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 1 to mira0435601-461 2014-07-01 16:18:03,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 1 to mira0435601-466 2014-07-01 16:18:03,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 3 to mira0435601-461 2014-07-01 16:18:03,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 3 to mira0435601-466 2014-07-01 16:18:03,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 1 to mira0435601-457 2014-07-01 16:18:03,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 3 to mira0435601-457 2014-07-01 16:18:03,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 6 to mira0435601-454 2014-07-01 16:18:03,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 74 (ObjNum 453 snap 0 seq_num 453) dirty exists 2014-07-01 16:18:03,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: done (15 left) 2014-07-01 16:18:03,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: write initial oid 467 2014-07-01 16:18:03,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: seq_num 466 ranges {0=421888} 2014-07-01 16:18:03,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: writing mira0435601-467 from 0 to 421888 tid 1 2014-07-01 16:18:03,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:03,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 4 to mira0435601-461 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 76 (ObjNum 460 snap 0 seq_num 460) dirty exists 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 4 to mira0435601-466 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 106 (ObjNum 465 snap 0 seq_num 465) dirty exists 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 4 to mira0435601-444 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 98 (ObjNum 443 snap 0 seq_num 443) dirty exists 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: done (15 left) 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: done (14 left) 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: done (13 left) 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: write initial oid 468 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: seq_num 467 ranges {0=290816} 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: writing mira0435601-468 from 0 to 290816 tid 1 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: write initial oid 469 2014-07-01 16:18:03,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: seq_num 468 ranges {0=720896,720896=421888,1142784=380928} 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: writing mira0435601-469 from 0 to 720896 tid 1 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: writing mira0435601-469 from 720896 to 1142784 tid 2 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: writing mira0435601-469 from 1142784 to 1523712 tid 3 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: write initial oid 470 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: seq_num 469 ranges {0=643072,643072=405504,1048576=557056,1605632=552960,2158592=90112} 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0435601-470 from 0 to 643072 tid 1 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0435601-470 from 643072 to 1048576 tid 2 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0435601-470 from 1048576 to 1605632 tid 3 2014-07-01 16:18:03,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0435601-470 from 1605632 to 2158592 tid 4 2014-07-01 16:18:03,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0435601-470 from 2158592 to 2248704 tid 5 2014-07-01 16:18:03,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 1 to mira0435601-455 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 1 to mira0435601-445 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 3 to mira0435601-445 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 4 to mira0435601-445 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 77 (ObjNum 444 snap 0 seq_num 444) dirty exists 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 4 to mira0435601-442 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 94 (ObjNum 441 snap 0 seq_num 441) dirty exists 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 1 to mira0435601-467 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 3 to mira0435601-467 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 4 to mira0435601-467 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 62 (ObjNum 466 snap 0 seq_num 466) dirty exists 2014-07-01 16:18:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 4 to mira0435601-457 2014-07-01 16:18:03,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 92 (ObjNum 456 snap 0 seq_num 456) dirty exists 2014-07-01 16:18:03,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 1 to mira0435601-464 2014-07-01 16:18:03,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 2 to mira0435601-464 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 1 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 1 to mira0435601-460 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 2 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 3 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 4 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 3 to mira0435601-464 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 4 to mira0435601-464 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 5 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 6 to mira0435601-464 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 1 to mira0435601-465 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 7 to mira0435601-462 2014-07-01 16:18:03,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 2 to mira0435601-465 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 2 to mira0435601-460 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 3 to mira0435601-460 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 4 to mira0435601-460 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 6 to mira0435601-460 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 1 to mira0435601-463 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 2 to mira0435601-463 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 4 to mira0435601-463 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 4 to mira0435601-465 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 7 to mira0435601-464 2014-07-01 16:18:03,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 126 (ObjNum 463 snap 0 seq_num 463) dirty exists 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 7 to mira0435601-460 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 83 (ObjNum 459 snap 0 seq_num 459) dirty exists 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 8 to mira0435601-462 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 84 (ObjNum 461 snap 0 seq_num 461) dirty exists 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 5 to mira0435601-465 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 465 v 129 (ObjNum 464 snap 0 seq_num 464) dirty exists 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 5 to mira0435601-463 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 65 (ObjNum 462 snap 0 seq_num 462) dirty exists 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 1 to mira0435601-470 2014-07-01 16:18:03,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: done (15 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: done (14 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: done (13 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: done (12 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: done (11 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: done (10 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: done (9 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: done (8 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: done (7 left) 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: write initial oid 471 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: seq_num 470 ranges {0=512000,512000=434176,946176=434176,1380352=659456,2039808=77824} 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0435601-471 from 0 to 512000 tid 1 2014-07-01 16:18:03,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0435601-471 from 512000 to 946176 tid 2 2014-07-01 16:18:03,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0435601-471 from 946176 to 1380352 tid 3 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0435601-471 from 1380352 to 2039808 tid 4 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0435601-471 from 2039808 to 2117632 tid 5 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: write initial oid 472 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: seq_num 471 ranges {0=540672,540672=401408} 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: writing mira0435601-472 from 0 to 540672 tid 1 2014-07-01 16:18:04,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: writing mira0435601-472 from 540672 to 942080 tid 2 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: write initial oid 473 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: seq_num 472 ranges {0=413696,413696=401408} 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: writing mira0435601-473 from 0 to 413696 tid 1 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: writing mira0435601-473 from 413696 to 815104 tid 2 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: write initial oid 474 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: seq_num 473 ranges {0=684032} 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: writing mira0435601-474 from 0 to 684032 tid 1 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: write initial oid 475 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: seq_num 474 ranges {0=360448} 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: writing mira0435601-475 from 0 to 360448 tid 1 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: write initial oid 476 2014-07-01 16:18:04,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: seq_num 475 ranges {0=233472} 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: writing mira0435601-476 from 0 to 233472 tid 1 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: write initial oid 477 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: seq_num 476 ranges {0=102400} 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: writing mira0435601-477 from 0 to 102400 tid 1 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: write initial oid 478 2014-07-01 16:18:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: seq_num 477 ranges {0=532480,532480=614400,1146880=188416} 2014-07-01 16:18:04,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: writing mira0435601-478 from 0 to 532480 tid 1 2014-07-01 16:18:04,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: writing mira0435601-478 from 532480 to 1146880 tid 2 2014-07-01 16:18:04,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: writing mira0435601-478 from 1146880 to 1335296 tid 3 2014-07-01 16:18:04,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: write initial oid 479 2014-07-01 16:18:04,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: seq_num 478 ranges {0=454656,454656=794624,1249280=729088,1978368=81920} 2014-07-01 16:18:04,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0435601-479 from 0 to 454656 tid 1 2014-07-01 16:18:04,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0435601-479 from 454656 to 1249280 tid 2 2014-07-01 16:18:04,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0435601-479 from 1249280 to 1978368 tid 3 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0435601-479 from 1978368 to 2060288 tid 4 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 2 to mira0435601-455 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 4 to mira0435601-455 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 1 to mira0435601-468 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 3 to mira0435601-468 2014-07-01 16:18:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 2 to mira0435601-470 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 5 to mira0435601-455 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 100 (ObjNum 454 snap 0 seq_num 454) dirty exists 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 3 to mira0435601-470 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 4 to mira0435601-468 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 468 v 102 (ObjNum 467 snap 0 seq_num 467) dirty exists 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 1 to mira0435601-446 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 2 to mira0435601-446 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: done (15 left) 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: done (14 left) 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: write initial oid 480 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: seq_num 479 ranges {0=479232,479232=401408} 2014-07-01 16:18:04,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: writing mira0435601-480 from 0 to 479232 tid 1 2014-07-01 16:18:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: writing mira0435601-480 from 479232 to 880640 tid 2 2014-07-01 16:18:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: write initial oid 481 2014-07-01 16:18:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: seq_num 480 ranges {0=753664} 2014-07-01 16:18:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: writing mira0435601-481 from 0 to 753664 tid 1 2014-07-01 16:18:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 3 to mira0435601-446 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 5 to mira0435601-446 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 6 to mira0435601-446 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 100 (ObjNum 445 snap 0 seq_num 445) dirty exists 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 4 to mira0435601-470 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 5 to mira0435601-470 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 7 to mira0435601-470 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 8 to mira0435601-470 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 110 (ObjNum 469 snap 0 seq_num 469) dirty exists 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 1 to mira0435601-472 2014-07-01 16:18:04,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 2 to mira0435601-472 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 4 to mira0435601-472 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 5 to mira0435601-472 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 93 (ObjNum 471 snap 0 seq_num 471) dirty exists 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 1 to mira0435601-473 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 2 to mira0435601-473 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 4 to mira0435601-473 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 5 to mira0435601-473 2014-07-01 16:18:04,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 113 (ObjNum 472 snap 0 seq_num 472) dirty exists 2014-07-01 16:18:04,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 1 to mira0435601-479 2014-07-01 16:18:04,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 2 to mira0435601-479 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: done (15 left) 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: done (14 left) 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: done (13 left) 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: done (12 left) 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: write initial oid 482 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: seq_num 481 ranges {0=626688} 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: writing mira0435601-482 from 0 to 626688 tid 1 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: write initial oid 483 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: seq_num 482 ranges {0=651264,651264=622592,1273856=581632} 2014-07-01 16:18:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0435601-483 from 0 to 651264 tid 1 2014-07-01 16:18:04,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0435601-483 from 651264 to 1273856 tid 2 2014-07-01 16:18:04,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0435601-483 from 1273856 to 1855488 tid 3 2014-07-01 16:18:04,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: write initial oid 484 2014-07-01 16:18:04,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: seq_num 483 ranges {0=172032} 2014-07-01 16:18:04,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: writing mira0435601-484 from 0 to 172032 tid 1 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: write initial oid 485 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: seq_num 484 ranges {0=45056} 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: writing mira0435601-485 from 0 to 45056 tid 1 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 3 to mira0435601-479 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 4 to mira0435601-479 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 6 to mira0435601-479 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 7 to mira0435601-479 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 115 (ObjNum 478 snap 0 seq_num 478) dirty exists 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 1 to mira0435601-476 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 3 to mira0435601-476 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 1 to mira0435601-480 2014-07-01 16:18:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 2 to mira0435601-480 2014-07-01 16:18:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 4 to mira0435601-480 2014-07-01 16:18:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 1 to mira0435601-471 2014-07-01 16:18:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 2 to mira0435601-471 2014-07-01 16:18:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 4 to mira0435601-476 2014-07-01 16:18:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 99 (ObjNum 475 snap 0 seq_num 475) dirty exists 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: done (15 left) 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: done (14 left) 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: write initial oid 486 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: seq_num 485 ranges {0=471040,471040=421888,892928=380928} 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: writing mira0435601-486 from 0 to 471040 tid 1 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: writing mira0435601-486 from 471040 to 892928 tid 2 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: writing mira0435601-486 from 892928 to 1273856 tid 3 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: write initial oid 487 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: seq_num 486 ranges {0=745472,745472=401408} 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: writing mira0435601-487 from 0 to 745472 tid 1 2014-07-01 16:18:04,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: writing mira0435601-487 from 745472 to 1146880 tid 2 2014-07-01 16:18:04,381.381 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 118 16 625 609 20.6395 30 8.17672 2.93154 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 5 to mira0435601-480 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 79 (ObjNum 479 snap 0 seq_num 479) dirty exists 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 1 to mira0435601-484 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 3 to mira0435601-484 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 4 to mira0435601-484 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 113 (ObjNum 483 snap 0 seq_num 483) dirty exists 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 1 to mira0435601-447 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 2 to mira0435601-447 2014-07-01 16:18:04,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: done (15 left) 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: done (14 left) 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: write initial oid 488 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: seq_num 487 ranges {0=663552,663552=585728,1249280=495616,1744896=122880} 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0435601-488 from 0 to 663552 tid 1 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0435601-488 from 663552 to 1249280 tid 2 2014-07-01 16:18:04,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0435601-488 from 1249280 to 1744896 tid 3 2014-07-01 16:18:04,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0435601-488 from 1744896 to 1867776 tid 4 2014-07-01 16:18:04,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: write initial oid 489 2014-07-01 16:18:04,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: seq_num 488 ranges {0=692224} 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: writing mira0435601-489 from 0 to 692224 tid 1 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 3 to mira0435601-447 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 5 to mira0435601-447 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 6 to mira0435601-447 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 104 (ObjNum 446 snap 0 seq_num 446) dirty exists 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 1 to mira0435601-469 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 2 to mira0435601-469 2014-07-01 16:18:04,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 3 to mira0435601-469 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 5 to mira0435601-469 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 6 to mira0435601-469 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 69 (ObjNum 468 snap 0 seq_num 468) dirty exists 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: done (15 left) 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: done (14 left) 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: write initial oid 490 2014-07-01 16:18:04,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: seq_num 489 ranges {0=720896,720896=802816,1523712=401408} 2014-07-01 16:18:04,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: writing mira0435601-490 from 0 to 720896 tid 1 2014-07-01 16:18:04,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: writing mira0435601-490 from 720896 to 1523712 tid 2 2014-07-01 16:18:04,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: writing mira0435601-490 from 1523712 to 1925120 tid 3 2014-07-01 16:18:04,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: write initial oid 491 2014-07-01 16:18:04,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: seq_num 490 ranges {0=434176} 2014-07-01 16:18:04,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: writing mira0435601-491 from 0 to 434176 tid 1 2014-07-01 16:18:04,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:04,546.546 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 16:18:05,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 119 16 626 610 20.4997 4 25.5529 2.96862 2014-07-01 16:18:06,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:18:06.200363min lat: 0.067553 max lat: 28.4511 avg lat: 3.01813 2014-07-01 16:18:06,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:18:06,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 120 16 629 613 20.4289 12 28.4511 3.01813 2014-07-01 16:18:07,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 121 16 631 615 20.3261 8 0.095393 3.01018 2014-07-01 16:18:08,288.288 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 122 16 631 615 20.1595 0 - 3.01018 2014-07-01 16:18:09,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 1 to mira0435601-439 2014-07-01 16:18:09,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 2 to mira0435601-439 2014-07-01 16:18:09,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 3 to mira0435601-439 2014-07-01 16:18:09,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 4 to mira0435601-439 2014-07-01 16:18:09,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 6 to mira0435601-439 2014-07-01 16:18:09,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 7 to mira0435601-439 2014-07-01 16:18:09,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 97 (ObjNum 438 snap 0 seq_num 438) dirty exists 2014-07-01 16:18:09,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: done (15 left) 2014-07-01 16:18:09,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: write initial oid 492 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: seq_num 491 ranges {0=462848,462848=569344,1032192=634880} 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0435601-492 from 0 to 462848 tid 1 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0435601-492 from 462848 to 1032192 tid 2 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0435601-492 from 1032192 to 1667072 tid 3 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 3 to mira0435601-471 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 4 to mira0435601-471 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 5 to mira0435601-471 2014-07-01 16:18:09,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 7 to mira0435601-471 2014-07-01 16:18:09,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 123 16 636 620 20.1582 10 5.27631 3.02236 2014-07-01 16:18:09,559.559 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:18:09,559.559 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:09,559.559 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.213226 2014-07-01 16:18:09,559.559 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0.213225703327' 2014-07-01 16:18:09,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 8 to mira0435601-471 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 135 (ObjNum 470 snap 0 seq_num 470) dirty exists 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: done (15 left) 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: write initial oid 493 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: seq_num 492 ranges {0=540672,540672=643072,1183744=159744} 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0435601-493 from 0 to 540672 tid 1 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0435601-493 from 540672 to 1183744 tid 2 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0435601-493 from 1183744 to 1343488 tid 3 2014-07-01 16:18:09,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:09,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 1 to mira0435601-492 2014-07-01 16:18:09,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 2 to mira0435601-492 2014-07-01 16:18:09,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 3 to mira0435601-492 2014-07-01 16:18:09,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 5 to mira0435601-492 2014-07-01 16:18:09,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 1 to mira0435601-485 2014-07-01 16:18:09,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 3 to mira0435601-485 2014-07-01 16:18:09,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 4 to mira0435601-485 2014-07-01 16:18:09,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 104 (ObjNum 484 snap 0 seq_num 484) dirty exists 2014-07-01 16:18:09,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: done (15 left) 2014-07-01 16:18:09,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: write initial oid 494 2014-07-01 16:18:09,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: seq_num 493 ranges {0=655360,655360=626688,1282048=688128,1970176=290816} 2014-07-01 16:18:09,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0435601-494 from 0 to 655360 tid 1 2014-07-01 16:18:10,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0435601-494 from 655360 to 1282048 tid 2 2014-07-01 16:18:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0435601-494 from 1282048 to 1970176 tid 3 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0435601-494 from 1970176 to 2260992 tid 4 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 1 to mira0435601-474 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 3 to mira0435601-474 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 4 to mira0435601-474 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 72 (ObjNum 473 snap 0 seq_num 473) dirty exists 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: done (15 left) 2014-07-01 16:18:10,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: write initial oid 495 2014-07-01 16:18:10,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: seq_num 494 ranges {0=684032,684032=401408} 2014-07-01 16:18:10,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: writing mira0435601-495 from 0 to 684032 tid 1 2014-07-01 16:18:10,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: writing mira0435601-495 from 684032 to 1085440 tid 2 2014-07-01 16:18:10,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:10,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 1 to mira0435601-483 2014-07-01 16:18:10,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 2 to mira0435601-483 2014-07-01 16:18:10,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 3 to mira0435601-483 2014-07-01 16:18:10,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 5 to mira0435601-483 2014-07-01 16:18:10,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 6 to mira0435601-483 2014-07-01 16:18:10,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 76 (ObjNum 482 snap 0 seq_num 482) dirty exists 2014-07-01 16:18:10,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: done (15 left) 2014-07-01 16:18:10,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: write initial oid 496 2014-07-01 16:18:10,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: seq_num 495 ranges {0=712704,712704=794624,1507328=790528,2297856=20480} 2014-07-01 16:18:10,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: writing mira0435601-496 from 0 to 712704 tid 1 2014-07-01 16:18:10,287.287 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 124 16 639 623 20.0924 12 0.556729 3.05452 2014-07-01 16:18:10,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: writing mira0435601-496 from 712704 to 1507328 tid 2 2014-07-01 16:18:10,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: writing mira0435601-496 from 1507328 to 2297856 tid 3 2014-07-01 16:18:10,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: writing mira0435601-496 from 2297856 to 2318336 tid 4 2014-07-01 16:18:10,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:10,681.681 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0.213226 (8139732) 2014-07-01 16:18:11,289.289 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 125 16 639 623 19.9316 0 - 3.05452 2014-07-01 16:18:12,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 126 16 639 623 19.7734 0 - 3.05452 2014-07-01 16:18:12,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 1 to mira0435601-486 2014-07-01 16:18:12,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 2 to mira0435601-486 2014-07-01 16:18:12,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 3 to mira0435601-486 2014-07-01 16:18:12,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 5 to mira0435601-486 2014-07-01 16:18:12,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 6 to mira0435601-486 2014-07-01 16:18:12,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 81 (ObjNum 485 snap 0 seq_num 485) dirty exists 2014-07-01 16:18:12,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: done (15 left) 2014-07-01 16:18:12,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: write initial oid 497 2014-07-01 16:18:12,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: seq_num 496 ranges {0=475136,475136=622592,1097728=458752,1556480=122880} 2014-07-01 16:18:12,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0435601-497 from 0 to 475136 tid 1 2014-07-01 16:18:12,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0435601-497 from 475136 to 1097728 tid 2 2014-07-01 16:18:12,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0435601-497 from 1097728 to 1556480 tid 3 2014-07-01 16:18:12,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0435601-497 from 1556480 to 1679360 tid 4 2014-07-01 16:18:12,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 1 to mira0435601-496 2014-07-01 16:18:12,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 6 to mira0435601-492 2014-07-01 16:18:12,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 80 (ObjNum 491 snap 0 seq_num 491) dirty exists 2014-07-01 16:18:12,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: done (15 left) 2014-07-01 16:18:12,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: write initial oid 498 2014-07-01 16:18:12,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: seq_num 497 ranges {0=503808} 2014-07-01 16:18:12,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: writing mira0435601-498 from 0 to 503808 tid 1 2014-07-01 16:18:12,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 2 to mira0435601-496 2014-07-01 16:18:12,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 3 to mira0435601-496 2014-07-01 16:18:12,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 4 to mira0435601-496 2014-07-01 16:18:12,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 6 to mira0435601-496 2014-07-01 16:18:12,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 1 to mira0435601-498 2014-07-01 16:18:12,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 3 to mira0435601-498 2014-07-01 16:18:12,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 4 to mira0435601-498 2014-07-01 16:18:12,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 96 (ObjNum 497 snap 0 seq_num 497) dirty exists 2014-07-01 16:18:12,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: done (15 left) 2014-07-01 16:18:12,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: write initial oid 499 2014-07-01 16:18:12,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: seq_num 498 ranges {0=532480,532480=790528,1323008=413696} 2014-07-01 16:18:12,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0435601-499 from 0 to 532480 tid 1 2014-07-01 16:18:12,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0435601-499 from 532480 to 1323008 tid 2 2014-07-01 16:18:12,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0435601-499 from 1323008 to 1736704 tid 3 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 7 to mira0435601-496 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 85 (ObjNum 495 snap 0 seq_num 495) dirty exists 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: done (15 left) 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: write initial oid 500 2014-07-01 16:18:12,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: seq_num 499 ranges {0=245760} 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: writing mira0435601-500 from 0 to 245760 tid 1 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 1 to mira0435601-481 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 3 to mira0435601-481 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 1 to mira0435601-475 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 1 to mira0435601-482 2014-07-01 16:18:12,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 3 to mira0435601-482 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 3 to mira0435601-475 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 4 to mira0435601-482 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 108 (ObjNum 481 snap 0 seq_num 481) dirty exists 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: done (15 left) 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 424 current snap is 0 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: seq_num 500 ranges {1110016=675840,1785856=602112,2387968=200704} 2014-07-01 16:18:12,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: writing mira0435601-424 from 1110016 to 1785856 tid 1 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: writing mira0435601-424 from 1785856 to 2387968 tid 2 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: writing mira0435601-424 from 2387968 to 2588672 tid 3 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 4 to mira0435601-475 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 106 (ObjNum 474 snap 0 seq_num 474) dirty exists 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: done (15 left) 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:502: read oid 230 snap -1 2014-07-01 16:18:12,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:12,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 1 to mira0435601-487 2014-07-01 16:18:13,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 1 to mira0435601-477 2014-07-01 16:18:13,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 3 to mira0435601-477 2014-07-01 16:18:13,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 1 to mira0435601-499 2014-07-01 16:18:13,289.289 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 127 16 652 636 20.0271 17.3333 8.75095 3.13046 2014-07-01 16:18:14,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 2 to mira0435601-499 2014-07-01 16:18:14,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 3 to mira0435601-499 2014-07-01 16:18:14,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 5 to mira0435601-499 2014-07-01 16:18:14,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 1 to mira0435601-497 2014-07-01 16:18:14,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 2 to mira0435601-497 2014-07-01 16:18:14,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 3 to mira0435601-497 2014-07-01 16:18:14,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 4 to mira0435601-497 2014-07-01 16:18:14,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 1 to mira0435601-489 2014-07-01 16:18:14,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 3 to mira0435601-489 2014-07-01 16:18:14,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 1 to mira0435601-494 2014-07-01 16:18:14,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 6 to mira0435601-497 2014-07-01 16:18:14,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 2 to mira0435601-494 2014-07-01 16:18:14,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 3 to mira0435601-494 2014-07-01 16:18:14,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 4 to mira0435601-494 2014-07-01 16:18:14,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 6 to mira0435601-494 2014-07-01 16:18:14,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 6 to mira0435601-499 2014-07-01 16:18:14,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 499 v 103 (ObjNum 498 snap 0 seq_num 498) dirty exists 2014-07-01 16:18:14,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: done (15 left) 2014-07-01 16:18:14,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: read oid 402 snap -1 2014-07-01 16:18:14,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:14,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 7 to mira0435601-494 2014-07-01 16:18:14,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 113 (ObjNum 493 snap 0 seq_num 493) dirty exists 2014-07-01 16:18:14,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: done (15 left) 2014-07-01 16:18:14,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 16:18:14,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: seq_num 501 ranges {1167360=753664,1921024=401408} 2014-07-01 16:18:14,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: writing mira0435601-58 from 1167360 to 1921024 tid 1 2014-07-01 16:18:14,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: writing mira0435601-58 from 1921024 to 2322432 tid 2 2014-07-01 16:18:14,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:14,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 4 to mira0435601-481 2014-07-01 16:18:14,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 67 (ObjNum 480 snap 0 seq_num 480) dirty exists 2014-07-01 16:18:14,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 4 to mira0435601-477 2014-07-01 16:18:14,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 66 (ObjNum 476 snap 0 seq_num 476) dirty exists 2014-07-01 16:18:14,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: done (15 left) 2014-07-01 16:18:14,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: done (14 left) 2014-07-01 16:18:14,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 203 current snap is 0 2014-07-01 16:18:14,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: seq_num 502 ranges {2093056=622592,2715648=401408} 2014-07-01 16:18:14,288.288 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 128 16 657 641 20.0269 20 5.22928 3.13695 2014-07-01 16:18:14,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: writing mira0435601-203 from 2093056 to 2715648 tid 1 2014-07-01 16:18:14,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: writing mira0435601-203 from 2715648 to 3117056 tid 2 2014-07-01 16:18:14,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 16:18:14,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: seq_num 503 ranges {593920=495616,1089536=401408} 2014-07-01 16:18:14,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: writing mira0435601-434 from 593920 to 1089536 tid 1 2014-07-01 16:18:14,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: writing mira0435601-434 from 1089536 to 1490944 tid 2 2014-07-01 16:18:14,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:14,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:502: expect (ObjNum 229 snap 0 seq_num 229) 2014-07-01 16:18:15,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:502: done (15 left) 2014-07-01 16:18:15,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 194 current snap is 0 2014-07-01 16:18:15,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: seq_num 504 ranges {2281472=770048} 2014-07-01 16:18:15,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: writing mira0435601-194 from 2281472 to 3051520 tid 1 2014-07-01 16:18:15,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:15,289.289 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 129 16 657 641 19.8716 0 - 3.13695 2014-07-01 16:18:15,691.691 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:18:15,692.692 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:15,692.692 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 16:18:15,693.693 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:18:16,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 130 16 657 641 19.7188 0 - 3.13695 2014-07-01 16:18:17,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 131 16 657 641 19.5682 0 - 3.13695 2014-07-01 16:18:18,291.291 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 132 16 657 641 19.42 0 - 3.13695 2014-07-01 16:18:18,354.354 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:18:19,258.258 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:18:19,258.258 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pg_num 34' 2014-07-01 16:18:19,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 133 16 660 644 19.3642 2.4 5.97973 3.14858 2014-07-01 16:18:20,290.290 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 134 16 660 644 19.2197 0 - 3.14858 2014-07-01 16:18:20,340.340 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pg_num to 34 2014-07-01 16:18:21,291.291 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 135 16 662 646 19.1366 4 11.2713 3.16877 2014-07-01 16:18:21,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 7 to mira0435601-497 2014-07-01 16:18:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 140 (ObjNum 496 snap 0 seq_num 496) dirty exists 2014-07-01 16:18:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: done (15 left) 2014-07-01 16:18:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: read oid 22 snap -1 2014-07-01 16:18:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:22,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 136 16 662 646 18.9959 0 - 3.16877 2014-07-01 16:18:22,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 1 to mira0435601-490 2014-07-01 16:18:22,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 2 to mira0435601-490 2014-07-01 16:18:22,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 1 to mira0435601-491 2014-07-01 16:18:22,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 1 to mira0435601-488 2014-07-01 16:18:22,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 3 to mira0435601-490 2014-07-01 16:18:22,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 3 to mira0435601-491 2014-07-01 16:18:22,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 4 to mira0435601-491 2014-07-01 16:18:22,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 108 (ObjNum 490 snap 0 seq_num 490) dirty exists 2014-07-01 16:18:22,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: done (15 left) 2014-07-01 16:18:22,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 446 current snap is 0 2014-07-01 16:18:22,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: seq_num 505 ranges {1380352=442368} 2014-07-01 16:18:22,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: writing mira0435601-446 from 1380352 to 1822720 tid 1 2014-07-01 16:18:22,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:22,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 1 to mira0435601-58 2014-07-01 16:18:22,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 1 to mira0435601-493 2014-07-01 16:18:22,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 5 to mira0435601-490 2014-07-01 16:18:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 6 to mira0435601-490 2014-07-01 16:18:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 73 (ObjNum 489 snap 0 seq_num 489) dirty exists 2014-07-01 16:18:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: done (15 left) 2014-07-01 16:18:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:18:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: seq_num 506 ranges {53248=315392} 2014-07-01 16:18:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: writing mira0435601-280 from 53248 to 368640 tid 1 2014-07-01 16:18:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:23,291.291 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 137 16 669 653 19.0615 14 4.20123 3.2243 2014-07-01 16:18:23,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 2 to mira0435601-58 2014-07-01 16:18:23,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 4 to mira0435601-58 2014-07-01 16:18:23,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 1 to mira0435601-495 2014-07-01 16:18:23,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 2 to mira0435601-495 2014-07-01 16:18:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 2 to mira0435601-488 2014-07-01 16:18:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 3 to mira0435601-488 2014-07-01 16:18:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 4 to mira0435601-488 2014-07-01 16:18:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 6 to mira0435601-488 2014-07-01 16:18:23,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 2 to mira0435601-493 2014-07-01 16:18:23,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 3 to mira0435601-493 2014-07-01 16:18:23,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 5 to mira0435601-493 2014-07-01 16:18:23,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 6 to mira0435601-493 2014-07-01 16:18:23,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 112 (ObjNum 492 snap 0 seq_num 492) dirty exists 2014-07-01 16:18:23,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: done (15 left) 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 190 current snap is 0 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: seq_num 507 ranges {1634304=184320} 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: writing mira0435601-190 from 1634304 to 1818624 tid 1 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 1 to mira0435601-478 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 2 to mira0435601-478 2014-07-01 16:18:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 3 to mira0435601-478 2014-07-01 16:18:23,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 5 to mira0435601-478 2014-07-01 16:18:23,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 6 to mira0435601-478 2014-07-01 16:18:23,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 478 v 76 (ObjNum 477 snap 0 seq_num 477) dirty exists 2014-07-01 16:18:23,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: done (15 left) 2014-07-01 16:18:23,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: read oid 152 snap -1 2014-07-01 16:18:23,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:23,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: expect (ObjNum 151 snap 0 seq_num 151) 2014-07-01 16:18:24,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 138 16 684 668 19.3581 60 0.429069 3.2313 2014-07-01 16:18:24,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: finishing write tid 1 to mira0435601-446 2014-07-01 16:18:24,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: finishing write tid 3 to mira0435601-446 2014-07-01 16:18:24,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 1 to mira0435601-434 2014-07-01 16:18:24,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 2 to mira0435601-434 2014-07-01 16:18:24,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 4 to mira0435601-434 2014-07-01 16:18:24,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: done (15 left) 2014-07-01 16:18:24,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: read oid 232 snap -1 2014-07-01 16:18:24,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 7 to mira0435601-488 2014-07-01 16:18:24,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 81 (ObjNum 487 snap 0 seq_num 487) dirty exists 2014-07-01 16:18:24,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: finishing write tid 4 to mira0435601-446 2014-07-01 16:18:24,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 114 (ObjNum 505 snap 0 seq_num 505) dirty exists 2014-07-01 16:18:24,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 5 to mira0435601-434 2014-07-01 16:18:24,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 101 (ObjNum 503 snap 0 seq_num 503) dirty exists 2014-07-01 16:18:24,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: done (15 left) 2014-07-01 16:18:24,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: done (14 left) 2014-07-01 16:18:24,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: done (13 left) 2014-07-01 16:18:24,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:514: read oid 122 snap -1 2014-07-01 16:18:24,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: read oid 392 snap -1 2014-07-01 16:18:24,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:516: read oid 423 snap -1 2014-07-01 16:18:24,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:24,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 4 to mira0435601-495 2014-07-01 16:18:24,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: expect (ObjNum 401 snap 0 seq_num 401) 2014-07-01 16:18:25,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 1 to mira0435601-500 2014-07-01 16:18:25,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 4 to mira0435601-489 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 91 (ObjNum 488 snap 0 seq_num 488) dirty exists 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 3 to mira0435601-500 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: finishing write tid 1 to mira0435601-424 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: finishing write tid 2 to mira0435601-424 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: finishing write tid 3 to mira0435601-424 2014-07-01 16:18:25,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: finishing write tid 5 to mira0435601-424 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 2 to mira0435601-487 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 4 to mira0435601-487 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: done (15 left) 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: done (14 left) 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 75 current snap is 0 2014-07-01 16:18:25,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: seq_num 508 ranges {917504=614400,1531904=409600,1941504=393216} 2014-07-01 16:18:25,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: writing mira0435601-75 from 917504 to 1531904 tid 1 2014-07-01 16:18:25,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: writing mira0435601-75 from 1531904 to 1941504 tid 2 2014-07-01 16:18:25,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: writing mira0435601-75 from 1941504 to 2334720 tid 3 2014-07-01 16:18:25,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 136 current snap is 0 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: seq_num 509 ranges {1404928=487424,1892352=794624,2686976=8192} 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: writing mira0435601-136 from 1404928 to 1892352 tid 1 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: writing mira0435601-136 from 1892352 to 2686976 tid 2 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: writing mira0435601-136 from 2686976 to 2695168 tid 3 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: finishing write tid 1 to mira0435601-203 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: finishing write tid 2 to mira0435601-203 2014-07-01 16:18:25,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: finishing write tid 4 to mira0435601-203 2014-07-01 16:18:25,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: finishing write tid 1 to mira0435601-194 2014-07-01 16:18:25,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: finishing write tid 3 to mira0435601-194 2014-07-01 16:18:25,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: expect (ObjNum 21 snap 0 seq_num 21) 2014-07-01 16:18:25,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 139 16 690 674 19.3915 24 1.60172 3.24185 2014-07-01 16:18:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 5 to mira0435601-58 2014-07-01 16:18:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 65 (ObjNum 501 snap 0 seq_num 501) dirty exists 2014-07-01 16:18:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: expect (ObjNum 231 snap 0 seq_num 231) 2014-07-01 16:18:25,357.357 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:18:25,357.357 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:25,357.357 INFO:teuthology.task.thrashosds.thrasher:Killing osd 5, live_osds are [1, 0, 3, 2, 5, 4] 2014-07-01 16:18:25,359.359 DEBUG:teuthology.task.ceph.osd.5:waiting for process to exit 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 5 to mira0435601-487 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 93 (ObjNum 486 snap 0 seq_num 486) dirty exists 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: finishing write tid 6 to mira0435601-424 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 97 (ObjNum 500 snap 0 seq_num 500) dirty exists 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 4 to mira0435601-500 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 500 v 118 (ObjNum 499 snap 0 seq_num 499) dirty exists 2014-07-01 16:18:25,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:514: expect (ObjNum 121 snap 0 seq_num 121) 2014-07-01 16:18:25,692.692 INFO:teuthology.task.radosbench.radosbench.0.mira043.stderr:2014-07-01 07:18:25.604344 7f8db1fef700 0 -- 10.214.133.128:0/1005625 >> 10.214.132.120:6810/6502 pipe(0x7f8dac024860 sd=12 :0 s=1 pgs=0 cs=0 l=1 c=0x7f8dac01fb70).fault 2014-07-01 16:18:25,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: done (15 left) 2014-07-01 16:18:25,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: done (14 left) 2014-07-01 16:18:25,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: done (13 left) 2014-07-01 16:18:25,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: done (12 left) 2014-07-01 16:18:25,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: done (11 left) 2014-07-01 16:18:25,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: done (10 left) 2014-07-01 16:18:25,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:514: done (9 left) 2014-07-01 16:18:25,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 493 current snap is 0 2014-07-01 16:18:25,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: seq_num 510 ranges {1343488=405504,1748992=577536,2326528=503808,2830336=479232,3309568=45056} 2014-07-01 16:18:25,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: writing mira0435601-493 from 1343488 to 1748992 tid 1 2014-07-01 16:18:25,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: writing mira0435601-493 from 1748992 to 2326528 tid 2 2014-07-01 16:18:25,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: writing mira0435601-493 from 2326528 to 2830336 tid 3 2014-07-01 16:18:26,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: writing mira0435601-493 from 2830336 to 3309568 tid 4 2014-07-01 16:18:26,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: writing mira0435601-493 from 3309568 to 3354624 tid 5 2014-07-01 16:18:26,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: read oid 215 snap -1 2014-07-01 16:18:26,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: read oid 339 snap -1 2014-07-01 16:18:26,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: delete oid 249 current snap is 0 2014-07-01 16:18:26,291.291 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:18:26.204557min lat: 0.067553 max lat: 28.4511 avg lat: 3.24023 2014-07-01 16:18:26,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:18:26,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 140 16 691 675 19.2815 4 2.14526 3.24023 2014-07-01 16:18:27,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 141 16 693 677 19.2015 8 2.63219 3.23852 2014-07-01 16:18:28,292.292 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 142 16 693 677 19.0663 0 - 3.23852 2014-07-01 16:18:29,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 143 16 693 677 18.9329 0 - 3.23852 2014-07-01 16:18:30,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 144 16 693 677 18.8015 0 - 3.23852 2014-07-01 16:18:31,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 145 16 693 677 18.6718 0 - 3.23852 2014-07-01 16:18:31,359.359 INFO:teuthology.task.ceph.osd.5:Stopped 2014-07-01 16:18:31,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: finishing write tid 5 to mira0435601-203 2014-07-01 16:18:31,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 116 (ObjNum 502 snap 0 seq_num 502) dirty exists 2014-07-01 16:18:31,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: finishing write tid 4 to mira0435601-194 2014-07-01 16:18:31,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 118 (ObjNum 504 snap 0 seq_num 504) dirty exists 2014-07-01 16:18:31,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 5 to mira0435601-495 2014-07-01 16:18:31,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 116 (ObjNum 494 snap 0 seq_num 494) dirty exists 2014-07-01 16:18:31,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:516: expect (ObjNum 422 snap 0 seq_num 422) 2014-07-01 16:18:32,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 146 16 704 688 18.8452 8.8 0.328798 3.31959 2014-07-01 16:18:33,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 147 16 710 694 18.8803 24 1.47156 3.30183 2014-07-01 16:18:33,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: expect (ObjNum 214 snap 0 seq_num 214) 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: finishing write tid 1 to mira0435601-280 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: finishing write tid 3 to mira0435601-280 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: finishing write tid 1 to mira0435601-75 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: finishing write tid 2 to mira0435601-75 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: finishing write tid 3 to mira0435601-75 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: finishing write tid 5 to mira0435601-75 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: finishing write tid 4 to mira0435601-280 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 120 (ObjNum 506 snap 0 seq_num 506) dirty exists 2014-07-01 16:18:34,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: finishing write tid 6 to mira0435601-75 2014-07-01 16:18:34,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 124 (ObjNum 508 snap 0 seq_num 508) dirty exists 2014-07-01 16:18:34,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 148 16 710 694 18.7527 0 - 3.30183 2014-07-01 16:18:35,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 149 16 710 694 18.6268 0 - 3.30183 2014-07-01 16:18:36,293.293 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 150 16 710 694 18.5027 0 - 3.30183 2014-07-01 16:18:36,359.359 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0] out_osds: [1] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:18:36,359.359 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:36,359.359 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 16:18:36,360.360 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 16:18:37,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 151 16 710 694 18.3801 0 - 3.30183 2014-07-01 16:18:38,295.295 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 152 16 710 694 18.2592 0 - 3.30183 2014-07-01 16:18:39,295.295 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 153 16 710 694 18.1399 0 - 3.30183 2014-07-01 16:18:40,294.294 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 154 16 710 694 18.0221 0 - 3.30183 2014-07-01 16:18:41,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 155 16 713 697 17.9832 1.5 8.2699 3.32208 2014-07-01 16:18:41,305.305 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 16:18:41,315.315 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 16:18:42,295.295 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 156 16 714 698 17.8936 4 1.4835 3.31945 2014-07-01 16:18:43,295.295 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 157 16 715 699 17.8051 4 2.11443 3.31772 2014-07-01 16:18:44,299.299 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 158 16 716 700 17.7177 4 1.03326 3.31446 2014-07-01 16:18:45,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 159 16 716 700 17.6062 0 - 3.31446 2014-07-01 16:18:46,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:18:46.208730min lat: 0.067553 max lat: 28.4511 avg lat: 3.31446 2014-07-01 16:18:46,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:18:46,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 160 16 716 700 17.4962 0 - 3.31446 2014-07-01 16:18:46,317.317 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:18:46,317.317 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:18:46,317.317 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 5 2014-07-01 16:18:46,317.317 INFO:teuthology.task.ceph.osd.5:Restarting daemon 2014-07-01 16:18:46,317.317 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 5' 2014-07-01 16:18:46,654.654 INFO:teuthology.task.ceph.osd.5:Started 2014-07-01 16:18:46,654.654 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok dump_ops_in_flight' 2014-07-01 16:18:46,712.712 INFO:teuthology.task.ceph.osd.5.mira028.stdout:starting osd.5 at :/0 osd_data /var/lib/ceph/osd/ceph-5 /var/lib/ceph/osd/ceph-5/journal 2014-07-01 16:18:46,782.782 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:18:46.710367 7f6764668780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:18:47,246.246 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:18:47.171759 7f6764668780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 60b8ea85/benchmark_data_mira043_5625_object558/head//4 in index: (2) No such file or directory 2014-07-01 16:18:47,246.246 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:18:47.171824 7f6764668780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find eb2078e5/benchmark_data_mira043_5625_object567/head//4 in index: (2) No such file or directory 2014-07-01 16:18:47,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 161 16 716 700 17.3875 0 - 3.31446 2014-07-01 16:18:48,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 162 16 716 700 17.2802 0 - 3.31446 2014-07-01 16:18:49,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 163 16 716 700 17.1742 0 - 3.31446 2014-07-01 16:18:50,296.296 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 164 16 716 700 17.0695 0 - 3.31446 2014-07-01 16:18:51,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 165 16 717 701 16.9903 0.571429 26.6461 3.34774 2014-07-01 16:18:51,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: finishing write tid 1 to mira0435601-136 2014-07-01 16:18:51,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: finishing write tid 2 to mira0435601-136 2014-07-01 16:18:51,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: finishing write tid 3 to mira0435601-136 2014-07-01 16:18:51,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: finishing write tid 5 to mira0435601-136 2014-07-01 16:18:51,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: finishing write tid 6 to mira0435601-136 2014-07-01 16:18:51,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 124 (ObjNum 509 snap 0 seq_num 509) dirty exists 2014-07-01 16:18:51,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 1 to mira0435601-493 2014-07-01 16:18:51,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 2 to mira0435601-493 2014-07-01 16:18:51,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 3 to mira0435601-493 2014-07-01 16:18:51,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 4 to mira0435601-493 2014-07-01 16:18:51,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 5 to mira0435601-493 2014-07-01 16:18:51,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 7 to mira0435601-493 2014-07-01 16:18:51,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: finishing write tid 8 to mira0435601-493 2014-07-01 16:18:51,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 120 (ObjNum 510 snap 0 seq_num 510) dirty exists 2014-07-01 16:18:52,297.297 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 166 16 721 705 16.9843 16 0.425506 3.36792 2014-07-01 16:18:53,297.297 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 167 16 722 706 16.9065 4 27.7905 3.40251 2014-07-01 16:18:54,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 168 16 726 710 16.9011 16 21.6827 3.48656 2014-07-01 16:18:54,781.781 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:18:54,782.782 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:18:55,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 169 16 727 711 16.8248 4 1.89317 3.48432 2014-07-01 16:18:55,626.626 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:18:55,639.639 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:18:56,299.299 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 170 16 727 711 16.7258 0 - 3.48432 2014-07-01 16:18:56,649.649 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:18:56,663.663 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:18:57,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 171 16 736 720 16.8385 18 5.48453 3.49475 2014-07-01 16:18:57,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: expect (ObjNum 391 snap 0 seq_num 391) 2014-07-01 16:18:58,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 172 16 741 725 16.8568 20 0.629455 3.58668 2014-07-01 16:18:58,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: expect (ObjNum 338 snap 0 seq_num 338) 2014-07-01 16:18:58,620.620 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:18:58,636.636 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:18:59,299.299 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 173 16 741 725 16.7594 0 - 3.58668 2014-07-01 16:18:59,658.658 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:18:59,673.673 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:19:00,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 174 16 742 726 16.686 2 2.42184 3.58507 2014-07-01 16:19:01,250.250 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:19:01,264.264 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:19:01,298.298 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 175 16 742 726 16.5907 0 - 3.58507 2014-07-01 16:19:02,293.293 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:19:02,299.299 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:19:02,299.299 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:02,300.300 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 176 16 754 738 16.7691 24 0.349378 3.68707 2014-07-01 16:19:02,984.984 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:03,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 177 16 762 746 16.8551 32 1.79985 3.72218 2014-07-01 16:19:04,112.112 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:04,299.299 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 178 16 763 747 16.7829 4 5.34036 3.72435 2014-07-01 16:19:05,029.029 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:05,301.301 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 179 16 772 756 16.8902 36 0.072153 3.70119 2014-07-01 16:19:06,056.056 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:06,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:19:06.212920min lat: 0.067553 max lat: 39.1344 avg lat: 3.6969 2014-07-01 16:19:06,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:19:06,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 180 15 780 765 16.9963 36 3.05871 3.6969 2014-07-01 16:19:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: done (12 left) 2014-07-01 16:19:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: done (11 left) 2014-07-01 16:19:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: done (10 left) 2014-07-01 16:19:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: done (9 left) 2014-07-01 16:19:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: done (8 left) 2014-07-01 16:19:06,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:516: done (7 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: done (6 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: done (5 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: done (4 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: done (3 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: done (2 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: done (1 left) 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:523: read oid 6 snap -1 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 61 current snap is 0 2014-07-01 16:19:06,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: seq_num 512 ranges {585728=708608} 2014-07-01 16:19:06,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: writing mira0435601-61 from 585728 to 1294336 tid 1 2014-07-01 16:19:06,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:525: delete oid 191 current snap is 0 2014-07-01 16:19:06,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:523: expect (ObjNum 5 snap 0 seq_num 5) 2014-07-01 16:19:06,864.864 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 8, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 8, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003191', u'skew': u'-0.013913', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002078', u'skew': u'-0.014466', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:18:56.757187', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884043700, u'health': u'HEALTH_OK', u'kb_used': 12737352, u'store_stats': {u'bytes_total': 7341929, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}, {u'last_updated': u'2014-07-01 07:18:54.527278', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887677832, u'health': u'HEALTH_OK', u'kb_used': 9103220, u'store_stats': {u'bytes_total': 8586601, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:19:01.656750', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887671248, u'health': u'HEALTH_OK', u'kb_used': 9109804, u'store_stats': {u'bytes_total': 9643412, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 267/2973 objects degraded (8.981%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 8097346, u'degraded_objects': 267, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 3625359196, u'degraded_total': 2973, u'bytes_used': 8621953024, u'recovering_objects_per_sec': 3, u'version': 146, u'pgs_by_state': [{u'count': 1, u'state_name': u'active'}, {u'count': 13, u'state_name': u'peering'}, {u'count': 116, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'8.981', u'bytes_avail': 5989677092864}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 70, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:06,865.865 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:07,300.300 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 181 16 784 768 16.9687 12 1.41225 3.6899 2014-07-01 16:19:07,684.684 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:19:07,684.684 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:08,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:523: done (3 left) 2014-07-01 16:19:08,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:525: done (2 left) 2014-07-01 16:19:08,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 372 current snap is 0 2014-07-01 16:19:08,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: seq_num 514 ranges {430080=253952} 2014-07-01 16:19:08,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: writing mira0435601-372 from 430080 to 684032 tid 1 2014-07-01 16:19:08,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 443 current snap is 0 2014-07-01 16:19:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: seq_num 515 ranges {405504=684032,1089536=786432,1875968=16384} 2014-07-01 16:19:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: writing mira0435601-443 from 405504 to 1089536 tid 1 2014-07-01 16:19:08,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 182 16 790 774 17.0073 24 1.94176 3.67521 2014-07-01 16:19:08,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: writing mira0435601-443 from 1089536 to 1875968 tid 2 2014-07-01 16:19:08,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: writing mira0435601-443 from 1875968 to 1892352 tid 3 2014-07-01 16:19:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:528: delete oid 33 current snap is 0 2014-07-01 16:19:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 1 to mira0435601-372 2014-07-01 16:19:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 3 to mira0435601-372 2014-07-01 16:19:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 4 to mira0435601-372 2014-07-01 16:19:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 99 (ObjNum 514 snap 0 seq_num 514) dirty exists 2014-07-01 16:19:08,511.511 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:09,339.339 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 183 16 795 779 17.0237 20 0.326661 3.6562 2014-07-01 16:19:10,301.301 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 184 16 795 779 16.9311 0 - 3.6562 2014-07-01 16:19:11,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 1 to mira0435601-61 2014-07-01 16:19:11,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 3 to mira0435601-61 2014-07-01 16:19:11,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 185 16 800 784 16.9477 10 4.84476 3.66846 2014-07-01 16:19:11,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 4 to mira0435601-61 2014-07-01 16:19:11,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 142 (ObjNum 512 snap 0 seq_num 512) dirty exists 2014-07-01 16:19:12,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 186 16 801 785 16.8781 4 5.04865 3.67022 2014-07-01 16:19:12,530.530 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:12,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: finishing write tid 1 to mira0435601-443 2014-07-01 16:19:12,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: finishing write tid 2 to mira0435601-443 2014-07-01 16:19:12,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: finishing write tid 3 to mira0435601-443 2014-07-01 16:19:12,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: finishing write tid 5 to mira0435601-443 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: done (4 left) 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: done (3 left) 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:528: done (2 left) 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: read oid 485 snap -1 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 16:19:12,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: seq_num 517 ranges {1261568=425984,1687552=602112,2289664=200704} 2014-07-01 16:19:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0435601-223 from 1261568 to 1687552 tid 1 2014-07-01 16:19:13,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0435601-223 from 1687552 to 2289664 tid 2 2014-07-01 16:19:13,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0435601-223 from 2289664 to 2490368 tid 3 2014-07-01 16:19:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:531: read oid 51 snap -1 2014-07-01 16:19:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 464 current snap is 0 2014-07-01 16:19:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: seq_num 518 ranges {2052096=53248} 2014-07-01 16:19:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: writing mira0435601-464 from 2052096 to 2105344 tid 1 2014-07-01 16:19:13,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 102 current snap is 0 2014-07-01 16:19:13,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: seq_num 519 ranges {348160=618496,966656=770048,1736704=434176} 2014-07-01 16:19:13,088.088 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:13,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: writing mira0435601-102 from 348160 to 966656 tid 1 2014-07-01 16:19:13,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: writing mira0435601-102 from 966656 to 1736704 tid 2 2014-07-01 16:19:13,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: writing mira0435601-102 from 1736704 to 2170880 tid 3 2014-07-01 16:19:13,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 233 current snap is 0 2014-07-01 16:19:13,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: seq_num 520 ranges {946176=647168} 2014-07-01 16:19:13,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: writing mira0435601-233 from 946176 to 1593344 tid 1 2014-07-01 16:19:13,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: read oid 109 snap -1 2014-07-01 16:19:13,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:19:13,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: finishing write tid 6 to mira0435601-443 2014-07-01 16:19:13,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 128 (ObjNum 515 snap 0 seq_num 515) dirty exists 2014-07-01 16:19:13,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:531: expect (ObjNum 50 snap 0 seq_num 50) 2014-07-01 16:19:13,300.300 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 187 16 818 802 17.1514 68 2.0574 3.72286 2014-07-01 16:19:14,153.153 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:14,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: finishing write tid 1 to mira0435601-233 2014-07-01 16:19:14,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: done (8 left) 2014-07-01 16:19:14,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: done (7 left) 2014-07-01 16:19:14,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:531: done (6 left) 2014-07-01 16:19:14,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 315 current snap is 0 2014-07-01 16:19:14,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: seq_num 521 ranges {1835008=675840,2510848=581632,3092480=413696,3506176=208896} 2014-07-01 16:19:14,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 188 16 827 811 17.2516 36 1.14766 3.69297 2014-07-01 16:19:14,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: writing mira0435601-315 from 1835008 to 2510848 tid 1 2014-07-01 16:19:14,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: writing mira0435601-315 from 2510848 to 3092480 tid 2 2014-07-01 16:19:14,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: writing mira0435601-315 from 3092480 to 3506176 tid 3 2014-07-01 16:19:14,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: writing mira0435601-315 from 3506176 to 3715072 tid 4 2014-07-01 16:19:14,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: read oid 20 snap -1 2014-07-01 16:19:14,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: finishing write tid 3 to mira0435601-233 2014-07-01 16:19:14,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: finishing write tid 4 to mira0435601-233 2014-07-01 16:19:14,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 77 (ObjNum 520 snap 0 seq_num 520) dirty exists 2014-07-01 16:19:14,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: expect (ObjNum 108 snap 0 seq_num 108) 2014-07-01 16:19:14,878.878 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 8, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 8, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003191', u'skew': u'-0.013913', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002078', u'skew': u'-0.014466', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:18:56.757187', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884043700, u'health': u'HEALTH_OK', u'kb_used': 12737352, u'store_stats': {u'bytes_total': 7341929, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}, {u'last_updated': u'2014-07-01 07:18:54.527278', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887677832, u'health': u'HEALTH_OK', u'kb_used': 9103220, u'store_stats': {u'bytes_total': 8586601, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:19:01.656750', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887671248, u'health': u'HEALTH_OK', u'kb_used': 9109804, u'store_stats': {u'bytes_total': 9643412, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'16 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 315/3113 objects degraded (10.119%); 3/1307 unfound (0.230%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'mon.c has mon_osd_down_out_interval set to 0'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 9917211, u'degraded_objects': 315, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 3909932892, u'degraded_total': 3113, u'bytes_used': 9588649984, u'unfound_ratio': u'0.230', u'recovering_objects_per_sec': 2, u'op_per_sec': 0, u'unfound_objects': 3, u'version': 151, u'pgs_by_state': [{u'count': 128, u'state_name': u'active+clean'}, {u'count': 16, u'state_name': u'active+recovering'}], u'read_bytes_sec': 807, u'degraded_ratio': u'10.119', u'bytes_avail': 5988710395904, u'unfound_total': 1307}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 70, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:14,879.879 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:15,301.301 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 189 16 831 815 17.245 16 0.839201 3.67809 2014-07-01 16:19:15,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: finishing write tid 1 to mira0435601-102 2014-07-01 16:19:15,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: finishing write tid 2 to mira0435601-102 2014-07-01 16:19:15,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: finishing write tid 3 to mira0435601-102 2014-07-01 16:19:15,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: finishing write tid 5 to mira0435601-102 2014-07-01 16:19:15,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: finishing write tid 6 to mira0435601-102 2014-07-01 16:19:15,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 132 (ObjNum 519 snap 0 seq_num 519) dirty exists 2014-07-01 16:19:15,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: done (7 left) 2014-07-01 16:19:15,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: done (6 left) 2014-07-01 16:19:15,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: done (5 left) 2014-07-01 16:19:15,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:538: read oid 334 snap -1 2014-07-01 16:19:15,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: finishing write tid 1 to mira0435601-464 2014-07-01 16:19:15,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: finishing write tid 3 to mira0435601-464 2014-07-01 16:19:15,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: finishing write tid 4 to mira0435601-464 2014-07-01 16:19:15,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 144 (ObjNum 518 snap 0 seq_num 518) dirty exists 2014-07-01 16:19:15,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: done (5 left) 2014-07-01 16:19:15,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: read oid 248 snap -1 2014-07-01 16:19:15,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: expect (ObjNum 19 snap 0 seq_num 19) 2014-07-01 16:19:15,583.583 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:19:15,583.583 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:16,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 190 16 831 815 17.1542 0 - 3.67809 2014-07-01 16:19:16,364.364 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:16,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 1 to mira0435601-315 2014-07-01 16:19:16,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 2 to mira0435601-315 2014-07-01 16:19:16,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 3 to mira0435601-315 2014-07-01 16:19:16,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 4 to mira0435601-315 2014-07-01 16:19:16,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 6 to mira0435601-315 2014-07-01 16:19:16,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: finishing write tid 7 to mira0435601-315 2014-07-01 16:19:16,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 101 (ObjNum 521 snap 0 seq_num 521) dirty exists 2014-07-01 16:19:16,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:538: expect (ObjNum 333 snap 0 seq_num 333) 2014-07-01 16:19:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: done (5 left) 2014-07-01 16:19:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: done (4 left) 2014-07-01 16:19:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:538: done (3 left) 2014-07-01 16:19:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:19:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: seq_num 522 ranges {2125824=389120} 2014-07-01 16:19:16,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: writing mira0435601-147 from 2125824 to 2514944 tid 1 2014-07-01 16:19:16,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 156 current snap is 0 2014-07-01 16:19:16,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: seq_num 523 ranges {1933312=65536} 2014-07-01 16:19:16,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: writing mira0435601-156 from 1933312 to 1998848 tid 1 2014-07-01 16:19:16,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:542: read oid 197 snap -1 2014-07-01 16:19:16,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:543: read oid 193 snap -1 2014-07-01 16:19:16,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: read oid 497 snap -1 2014-07-01 16:19:16,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 233 current snap is 0 2014-07-01 16:19:16,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: seq_num 524 ranges {1593344=495616,2088960=421888,2510848=380928} 2014-07-01 16:19:16,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: writing mira0435601-233 from 1593344 to 2088960 tid 1 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: writing mira0435601-233 from 2088960 to 2510848 tid 2 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: writing mira0435601-233 from 2510848 to 2891776 tid 3 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 293 current snap is 0 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: seq_num 525 ranges {516096=610304,1126400=405504,1531904=462848,1994752=737280} 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0435601-293 from 516096 to 1126400 tid 1 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0435601-293 from 1126400 to 1531904 tid 2 2014-07-01 16:19:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0435601-293 from 1531904 to 1994752 tid 3 2014-07-01 16:19:17,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0435601-293 from 1994752 to 2732032 tid 4 2014-07-01 16:19:17,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: read oid 141 snap -1 2014-07-01 16:19:17,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 209 current snap is 0 2014-07-01 16:19:17,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: seq_num 526 ranges {929792=638976,1568768=401408} 2014-07-01 16:19:17,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: writing mira0435601-209 from 929792 to 1568768 tid 1 2014-07-01 16:19:17,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: writing mira0435601-209 from 1568768 to 1970176 tid 2 2014-07-01 16:19:17,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:549: delete oid 4 current snap is 0 2014-07-01 16:19:17,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: finishing write tid 1 to mira0435601-156 2014-07-01 16:19:17,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: finishing write tid 3 to mira0435601-156 2014-07-01 16:19:17,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: finishing write tid 4 to mira0435601-156 2014-07-01 16:19:17,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 79 (ObjNum 523 snap 0 seq_num 523) dirty exists 2014-07-01 16:19:17,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:543: expect (ObjNum 192 snap 0 seq_num 192) 2014-07-01 16:19:17,301.301 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 191 16 831 815 17.0644 0 - 3.67809 2014-07-01 16:19:17,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:542: expect (ObjNum 196 snap 0 seq_num 196) 2014-07-01 16:19:18,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 192 16 836 820 17.0797 6.66667 4.70249 3.68621 2014-07-01 16:19:18,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: finishing write tid 1 to mira0435601-233 2014-07-01 16:19:18,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: finishing write tid 2 to mira0435601-233 2014-07-01 16:19:18,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: finishing write tid 3 to mira0435601-233 2014-07-01 16:19:18,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: finishing write tid 5 to mira0435601-233 2014-07-01 16:19:18,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: finishing write tid 6 to mira0435601-233 2014-07-01 16:19:18,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 83 (ObjNum 524 snap 0 seq_num 524) dirty exists 2014-07-01 16:19:18,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: expect (ObjNum 496 snap 0 seq_num 496) 2014-07-01 16:19:19,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 193 16 843 827 17.1362 28 4.34122 3.69095 2014-07-01 16:19:19,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: finishing write tid 1 to mira0435601-209 2014-07-01 16:19:19,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: finishing write tid 2 to mira0435601-209 2014-07-01 16:19:19,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: finishing write tid 4 to mira0435601-209 2014-07-01 16:19:19,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: finishing write tid 5 to mira0435601-209 2014-07-01 16:19:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 209 v 86 (ObjNum 526 snap 0 seq_num 526) dirty exists 2014-07-01 16:19:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 1 to mira0435601-223 2014-07-01 16:19:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 2 to mira0435601-223 2014-07-01 16:19:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 3 to mira0435601-223 2014-07-01 16:19:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 5 to mira0435601-223 2014-07-01 16:19:19,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 1 to mira0435601-293 2014-07-01 16:19:19,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: finishing write tid 1 to mira0435601-147 2014-07-01 16:19:19,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: expect (ObjNum 140 snap 0 seq_num 140) 2014-07-01 16:19:20,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: expect (ObjNum 247 snap 0 seq_num 247) 2014-07-01 16:19:20,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 6 to mira0435601-223 2014-07-01 16:19:20,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 106 (ObjNum 517 snap 0 seq_num 517) dirty exists 2014-07-01 16:19:20,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 2 to mira0435601-293 2014-07-01 16:19:20,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 3 to mira0435601-293 2014-07-01 16:19:20,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 4 to mira0435601-293 2014-07-01 16:19:20,243.243 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:20,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: finishing write tid 3 to mira0435601-147 2014-07-01 16:19:20,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: finishing write tid 4 to mira0435601-147 2014-07-01 16:19:20,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 68 (ObjNum 522 snap 0 seq_num 522) dirty exists 2014-07-01 16:19:20,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 194 16 849 833 17.1716 24 0.623209 3.66886 2014-07-01 16:19:20,769.769 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:21,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: done (12 left) 2014-07-01 16:19:21,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: done (11 left) 2014-07-01 16:19:21,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: done (10 left) 2014-07-01 16:19:21,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: done (9 left) 2014-07-01 16:19:21,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:542: done (8 left) 2014-07-01 16:19:21,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:543: done (7 left) 2014-07-01 16:19:21,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: done (6 left) 2014-07-01 16:19:21,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: done (5 left) 2014-07-01 16:19:21,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: done (4 left) 2014-07-01 16:19:21,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: done (3 left) 2014-07-01 16:19:21,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:549: done (2 left) 2014-07-01 16:19:21,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 6 current snap is 0 2014-07-01 16:19:21,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: seq_num 528 ranges {487424=585728} 2014-07-01 16:19:21,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: writing mira0435601-6 from 487424 to 1073152 tid 1 2014-07-01 16:19:21,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 477 current snap is 0 2014-07-01 16:19:21,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: seq_num 529 ranges {102400=458752} 2014-07-01 16:19:21,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: writing mira0435601-477 from 102400 to 561152 tid 1 2014-07-01 16:19:21,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:552: read oid 339 snap -1 2014-07-01 16:19:21,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: read oid 57 snap -1 2014-07-01 16:19:21,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:554: read oid 166 snap -1 2014-07-01 16:19:21,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:555: read oid 98 snap -1 2014-07-01 16:19:21,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: delete oid 88 current snap is 0 2014-07-01 16:19:21,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: done (8 left) 2014-07-01 16:19:21,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:557: read oid 133 snap -1 2014-07-01 16:19:21,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:558: delete oid 403 current snap is 0 2014-07-01 16:19:21,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:555: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 16:19:21,323.323 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 195 16 853 837 17.1656 16 6.67934 3.68599 2014-07-01 16:19:21,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:557: expect (ObjNum 132 snap 0 seq_num 132) 2014-07-01 16:19:21,809.809 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:21,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: expect (ObjNum 56 snap 0 seq_num 56) 2014-07-01 16:19:22,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 196 16 853 837 17.078 0 - 3.68599 2014-07-01 16:19:23,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 197 16 853 837 16.9913 0 - 3.68599 2014-07-01 16:19:23,808.808 INFO:teuthology.orchestra.run.mira028.stderr:start: Job is already running: ceph-all 2014-07-01 16:19:23,860.860 INFO:teuthology.orchestra.run.mira028.stderr:start: Job is already running: radosgw-all 2014-07-01 16:19:23,929.929 INFO:teuthology.orchestra.run.mira028.stderr:start: Job is already running: ceph-mds-all 2014-07-01 16:19:24,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 6 to mira0435601-293 2014-07-01 16:19:24,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:552: expect (ObjNum 338 snap 0 seq_num 338) 2014-07-01 16:19:24,118.118 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 8, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 8, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.003191', u'skew': u'-0.013913', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002078', u'skew': u'-0.014466', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:18:56.757187', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884043700, u'health': u'HEALTH_OK', u'kb_used': 12737352, u'store_stats': {u'bytes_total': 7341929, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1881}}, {u'last_updated': u'2014-07-01 07:18:54.527278', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887677832, u'health': u'HEALTH_OK', u'kb_used': 9103220, u'store_stats': {u'bytes_total': 8586601, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:19:01.656750', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887671248, u'health': u'HEALTH_OK', u'kb_used': 9109804, u'store_stats': {u'bytes_total': 9643412, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'7 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 246/3194 objects degraded (7.702%); 1/1349 unfound (0.074%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 19940927, u'degraded_objects': 246, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 4108207964, u'degraded_total': 3194, u'bytes_used': 10142523392, u'unfound_ratio': u'0.074', u'recovering_objects_per_sec': 4, u'write_bytes_sec': 0, u'op_per_sec': 7, u'unfound_objects': 1, u'version': 155, u'pgs_by_state': [{u'count': 136, u'state_name': u'active+clean'}, {u'count': 7, u'state_name': u'active+recovering'}, {u'count': 1, u'state_name': u'active+clean+scrubbing'}], u'read_bytes_sec': 1443920, u'degraded_ratio': u'7.702', u'bytes_avail': 5988156522496, u'unfound_total': 1349}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 70, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:24,118.118 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:24,167.167 INFO:teuthology.run_tasks:Running task ceph.restart... 2014-07-01 16:19:24,168.168 DEBUG:teuthology.task.ceph.mon.c:waiting for process to exit 2014-07-01 16:19:24,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 198 16 861 845 17.0671 10.6667 6.01787 3.70809 2014-07-01 16:19:24,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:554: expect (ObjNum 165 snap 0 seq_num 165) 2014-07-01 16:19:24,880.880 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:19:24,880.880 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:25,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 1 to mira0435601-6 2014-07-01 16:19:25,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 3 to mira0435601-6 2014-07-01 16:19:25,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: finishing write tid 1 to mira0435601-477 2014-07-01 16:19:25,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: finishing write tid 3 to mira0435601-477 2014-07-01 16:19:25,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 7 to mira0435601-293 2014-07-01 16:19:25,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 90 (ObjNum 525 snap 0 seq_num 525) dirty exists 2014-07-01 16:19:25,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: done (9 left) 2014-07-01 16:19:25,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:552: done (8 left) 2014-07-01 16:19:25,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: done (7 left) 2014-07-01 16:19:25,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:554: done (6 left) 2014-07-01 16:19:25,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:555: done (5 left) 2014-07-01 16:19:25,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:557: done (4 left) 2014-07-01 16:19:25,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:558: done (3 left) 2014-07-01 16:19:25,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: read oid 34 snap -1 2014-07-01 16:19:25,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 334 current snap is 0 2014-07-01 16:19:25,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: seq_num 532 ranges {278528=679936,958464=786432,1744896=684032,2428928=135168} 2014-07-01 16:19:25,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: writing mira0435601-334 from 278528 to 958464 tid 1 2014-07-01 16:19:25,302.302 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 199 16 870 854 17.1622 36 0.216764 3.7013 2014-07-01 16:19:25,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: writing mira0435601-334 from 958464 to 1744896 tid 2 2014-07-01 16:19:25,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: writing mira0435601-334 from 1744896 to 2428928 tid 3 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: writing mira0435601-334 from 2428928 to 2564096 tid 4 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: read oid 59 snap -1 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 160 current snap is 0 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: seq_num 533 ranges {1224704=704512,1929216=401408} 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: writing mira0435601-160 from 1224704 to 1929216 tid 1 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: writing mira0435601-160 from 1929216 to 2330624 tid 2 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:563: read oid 161 snap -1 2014-07-01 16:19:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: expect (ObjNum 33 snap 0 seq_num 33) 2014-07-01 16:19:25,503.503 INFO:teuthology.orchestra.run.mira079.stderr:2014-07-01 07:19:25.410725 7f9b8832c700 0 -- :/1009537 >> 10.214.132.120:6789/0 pipe(0x7f9b840155b0 sd=7 :0 s=1 pgs=0 cs=0 l=1 c=0x7f9b84015820).fault 2014-07-01 16:19:26,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:19:26.216304min lat: 0.067553 max lat: 48.8206 avg lat: 3.7013 2014-07-01 16:19:26,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:19:26,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 200 16 870 854 17.0764 0 - 3.7013 2014-07-01 16:19:26,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 1 to mira0435601-334 2014-07-01 16:19:26,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 2 to mira0435601-334 2014-07-01 16:19:26,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 3 to mira0435601-334 2014-07-01 16:19:26,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 4 to mira0435601-334 2014-07-01 16:19:26,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 6 to mira0435601-334 2014-07-01 16:19:26,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: finishing write tid 7 to mira0435601-334 2014-07-01 16:19:26,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 65 (ObjNum 532 snap 0 seq_num 532) dirty exists 2014-07-01 16:19:26,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: expect (ObjNum 58 snap 0 seq_num 58) 2014-07-01 16:19:27,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 201 16 870 854 16.9914 0 - 3.7013 2014-07-01 16:19:28,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 202 16 870 854 16.9073 0 - 3.7013 2014-07-01 16:19:28,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: finishing write tid 1 to mira0435601-160 2014-07-01 16:19:28,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: finishing write tid 2 to mira0435601-160 2014-07-01 16:19:28,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: finishing write tid 4 to mira0435601-160 2014-07-01 16:19:28,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: finishing write tid 5 to mira0435601-160 2014-07-01 16:19:28,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 89 (ObjNum 533 snap 0 seq_num 533) dirty exists 2014-07-01 16:19:28,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: done (7 left) 2014-07-01 16:19:28,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: done (6 left) 2014-07-01 16:19:28,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: done (5 left) 2014-07-01 16:19:28,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: done (4 left) 2014-07-01 16:19:28,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:564: read oid 152 snap -1 2014-07-01 16:19:28,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: read oid 174 snap -1 2014-07-01 16:19:28,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 235 current snap is 0 2014-07-01 16:19:28,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: seq_num 534 ranges {999424=577536,1576960=401408} 2014-07-01 16:19:28,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: writing mira0435601-235 from 999424 to 1576960 tid 1 2014-07-01 16:19:28,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: writing mira0435601-235 from 1576960 to 1978368 tid 2 2014-07-01 16:19:28,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:567: read oid 409 snap -1 2014-07-01 16:19:28,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:568: read oid 70 snap -1 2014-07-01 16:19:28,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:569: read oid 368 snap -1 2014-07-01 16:19:28,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 95 current snap is 0 2014-07-01 16:19:28,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: seq_num 535 ranges {1642496=450560,2093056=401408} 2014-07-01 16:19:28,779.779 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:28,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: writing mira0435601-95 from 1642496 to 2093056 tid 1 2014-07-01 16:19:28,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: writing mira0435601-95 from 2093056 to 2494464 tid 2 2014-07-01 16:19:28,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: read oid 80 snap -1 2014-07-01 16:19:28,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: finishing write tid 1 to mira0435601-235 2014-07-01 16:19:28,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:568: expect (ObjNum 69 snap 0 seq_num 69) 2014-07-01 16:19:28,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:569: expect (ObjNum 367 snap 0 seq_num 367) 2014-07-01 16:19:29,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 203 16 880 864 17.0211 10 0.141354 3.71778 2014-07-01 16:19:30,166.166 INFO:teuthology.task.ceph.mon.c:Stopped 2014-07-01 16:19:30,166.166 INFO:teuthology.task.ceph.mon.c:Restarting daemon 2014-07-01 16:19:30,167.167 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i c' 2014-07-01 16:19:30,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 204 16 880 864 16.9376 0 - 3.71778 2014-07-01 16:19:30,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: finishing write tid 2 to mira0435601-235 2014-07-01 16:19:30,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: finishing write tid 4 to mira0435601-235 2014-07-01 16:19:30,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: finishing write tid 5 to mira0435601-235 2014-07-01 16:19:30,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 123 (ObjNum 534 snap 0 seq_num 534) dirty exists 2014-07-01 16:19:30,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:567: expect (ObjNum 408 snap 0 seq_num 408) 2014-07-01 16:19:30,536.536 INFO:teuthology.task.ceph.mon.c:Started 2014-07-01 16:19:30,536.536 INFO:teuthology.task.ceph:Waiting until ceph osds are all up... 2014-07-01 16:19:30,536.536 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-07-01 16:19:31,054.054 DEBUG:teuthology.misc:6 of 6 OSDs are up 2014-07-01 16:19:31,054.054 INFO:teuthology.run_tasks:Running task ceph.wait_for_mon_quorum... 2014-07-01 16:19:31,055.055 INFO:teuthology.orchestra.run.mira079:Running: 'ceph quorum_status' 2014-07-01 16:19:31,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 205 16 880 864 16.855 0 - 3.71778 2014-07-01 16:19:31,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 1 to mira0435601-95 2014-07-01 16:19:31,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 2 to mira0435601-95 2014-07-01 16:19:31,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 4 to mira0435601-95 2014-07-01 16:19:31,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 5 to mira0435601-95 2014-07-01 16:19:31,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 104 (ObjNum 535 snap 0 seq_num 535) dirty exists 2014-07-01 16:19:31,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: expect (ObjNum 79 snap 0 seq_num 79) 2014-07-01 16:19:31,880.880 DEBUG:teuthology.task.ceph:Quorum: [u'c', u'a', u'b'] 2014-07-01 16:19:31,880.880 INFO:teuthology.run_tasks:Running task workunit... 2014-07-01 16:19:31,881.881 INFO:teuthology.task.workunit:Pulling workunits from ref firefly 2014-07-01 16:19:31,881.881 INFO:teuthology.task.workunit:Making a separate scratch dir for every client... 2014-07-01 16:19:31,881.881 DEBUG:teuthology.task.workunit:getting remote for 0 role client.0 2014-07-01 16:19:31,881.881 INFO:teuthology.orchestra.run.mira043:Running: 'stat -- /home/ubuntu/cephtest/mnt.0' 2014-07-01 16:19:32,251.251 INFO:teuthology.orchestra.run.mira043.stderr:stat: cannot stat `/home/ubuntu/cephtest/mnt.0': No such file or directory 2014-07-01 16:19:32,252.252 INFO:teuthology.orchestra.run.mira043:Running: 'mkdir -- /home/ubuntu/cephtest/mnt.0' 2014-07-01 16:19:32,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 206 16 889 873 16.9479 12 0.157756 3.72462 2014-07-01 16:19:32,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 4 to mira0435601-6 2014-07-01 16:19:32,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 75 (ObjNum 528 snap 0 seq_num 528) dirty exists 2014-07-01 16:19:32,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: finishing write tid 4 to mira0435601-477 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 70 (ObjNum 529 snap 0 seq_num 529) dirty exists 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: done (11 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: done (10 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: done (9 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:567: done (8 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:568: done (7 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:569: done (6 left) 2014-07-01 16:19:32,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: done (5 left) 2014-07-01 16:19:32,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: done (4 left) 2014-07-01 16:19:32,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 328 current snap is 0 2014-07-01 16:19:32,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: seq_num 536 ranges {2293760=475136,2768896=610304,3379200=528384,3907584=466944} 2014-07-01 16:19:32,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0435601-328 from 2293760 to 2768896 tid 1 2014-07-01 16:19:32,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0435601-328 from 2768896 to 3379200 tid 2 2014-07-01 16:19:32,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0435601-328 from 3379200 to 3907584 tid 3 2014-07-01 16:19:32,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0435601-328 from 3907584 to 4374528 tid 4 2014-07-01 16:19:32,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:573: read oid 142 snap -1 2014-07-01 16:19:32,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: expect (ObjNum 173 snap 0 seq_num 173) 2014-07-01 16:19:32,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:564: expect (ObjNum 151 snap 0 seq_num 151) 2014-07-01 16:19:32,649.649 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:32,758.758 INFO:teuthology.task.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2014-07-01 16:19:32,758.758 INFO:teuthology.orchestra.run.mira043:Running: 'cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0' 2014-07-01 16:19:33,196.196 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:33,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:563: expect (ObjNum 160 snap 0 seq_num 160) 2014-07-01 16:19:33,296.296 INFO:teuthology.orchestra.run.mira043:Running: "mkdir -- /home/ubuntu/cephtest/workunit.client.0 && git archive --remote=git://ceph.newdream.net/git/ceph.git firefly:qa/workunits | tar -C /home/ubuntu/cephtest/workunit.client.0 -x -f- && cd -- /home/ubuntu/cephtest/workunit.client.0 && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\\0' >/home/ubuntu/cephtest/workunits.list" 2014-07-01 16:19:33,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 207 16 897 881 17.0206 32 1.17783 3.7066 2014-07-01 16:19:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 1 to mira0435601-328 2014-07-01 16:19:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 2 to mira0435601-328 2014-07-01 16:19:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:573: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 16:19:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 3 to mira0435601-328 2014-07-01 16:19:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 4 to mira0435601-328 2014-07-01 16:19:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 6 to mira0435601-328 2014-07-01 16:19:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 7 to mira0435601-328 2014-07-01 16:19:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 110 (ObjNum 536 snap 0 seq_num 536) dirty exists 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:563: done (5 left) 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:564: done (4 left) 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: done (3 left) 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: done (2 left) 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:573: done (1 left) 2014-07-01 16:19:33,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:574: read oid 64 snap -1 2014-07-01 16:19:33,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:575: read oid 68 snap -1 2014-07-01 16:19:33,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: read oid 404 snap -1 2014-07-01 16:19:33,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: read oid 179 snap -1 2014-07-01 16:19:33,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:578: read oid 28 snap -1 2014-07-01 16:19:33,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: read oid 14 snap -1 2014-07-01 16:19:33,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:580: read oid 427 snap -1 2014-07-01 16:19:33,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 3 current snap is 0 2014-07-01 16:19:33,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: seq_num 537 ranges {1069056=397312} 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: writing mira0435601-3 from 1069056 to 1466368 tid 1 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: seq_num 538 ranges {622592=270336} 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: writing mira0435601-124 from 622592 to 892928 tid 1 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: read oid 388 snap -1 2014-07-01 16:19:33,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: expect (ObjNum 403 snap 0 seq_num 403) 2014-07-01 16:19:34,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:580: expect (ObjNum 426 snap 0 seq_num 426) 2014-07-01 16:19:34,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:574: expect (ObjNum 63 snap 0 seq_num 63) 2014-07-01 16:19:34,237.237 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:34,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 208 16 905 889 17.0926 32 1.54653 3.68444 2014-07-01 16:19:34,983.983 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'7 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 219/3249 objects degraded (6.741%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 219, u'num_pgs': 144, u'data_bytes': 4234549084, u'degraded_total': 3249, u'bytes_used': 10310270976, u'op_per_sec': 0, u'write_bytes_sec': 2905652, u'version': 161, u'pgs_by_state': [{u'count': 137, u'state_name': u'active+clean'}, {u'count': 7, u'state_name': u'active+recovering'}], u'degraded_ratio': u'6.741', u'bytes_avail': 5987988774912}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 73, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:34,984.984 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:35,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: expect (ObjNum 13 snap 0 seq_num 13) 2014-07-01 16:19:35,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:578: expect (ObjNum 27 snap 0 seq_num 27) 2014-07-01 16:19:35,303.303 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 209 16 912 896 17.1448 28 0.604192 3.70717 2014-07-01 16:19:35,644.644 INFO:teuthology.task.workunit.client.0.mira043.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2014-07-01 16:19:35,646.646 INFO:teuthology.task.workunit.client.0.mira043.stdout:make[1]: Entering directory `/home/ubuntu/cephtest/workunit.client.0/direct_io' 2014-07-01 16:19:35,647.647 INFO:teuthology.task.workunit.client.0.mira043.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2014-07-01 16:19:35,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: expect (ObjNum 387 snap 0 seq_num 387) 2014-07-01 16:19:35,754.754 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:19:35,755.755 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:35,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:575: expect (ObjNum 67 snap 0 seq_num 67) 2014-07-01 16:19:36,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 210 16 913 897 17.0822 4 2.00428 3.70528 2014-07-01 16:19:36,434.434 INFO:teuthology.task.workunit.client.0.mira043.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2014-07-01 16:19:36,490.490 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:36,531.531 INFO:teuthology.task.workunit.client.0.mira043.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2014-07-01 16:19:36,570.570 INFO:teuthology.task.workunit.client.0.mira043.stdout:make[1]: Leaving directory `/home/ubuntu/cephtest/workunit.client.0/direct_io' 2014-07-01 16:19:36,572.572 INFO:teuthology.task.workunit.client.0.mira043.stdout:make[1]: Entering directory `/home/ubuntu/cephtest/workunit.client.0/fs' 2014-07-01 16:19:36,572.572 INFO:teuthology.task.workunit.client.0.mira043.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2014-07-01 16:19:36,620.620 INFO:teuthology.task.workunit.client.0.mira043.stdout:make[1]: Leaving directory `/home/ubuntu/cephtest/workunit.client.0/fs' 2014-07-01 16:19:37,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 211 16 930 914 17.3234 68 0.175994 3.68623 2014-07-01 16:19:37,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: expect (ObjNum 178 snap 0 seq_num 178) 2014-07-01 16:19:38,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 212 16 936 920 17.3549 24 0.179515 3.6658 2014-07-01 16:19:38,468.468 INFO:teuthology.task.workunit:Running workunits matching rbd/test_librbd_python.sh on client.0... 2014-07-01 16:19:38,468.468 INFO:teuthology.task.workunit:Running workunit rbd/test_librbd_python.sh... 2014-07-01 16:19:38,468.468 INFO:teuthology.orchestra.run.mira043:Running: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=firefly TESTDIR="/home/ubuntu/cephtest" CEPH_ID="0" adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/workunit.client.0/rbd/test_librbd_python.sh' 2014-07-01 16:19:38,864.864 INFO:teuthology.task.workunit.client.0.mira043.stderr:+ CEPH_REF=firefly 2014-07-01 16:19:38,864.864 INFO:teuthology.task.workunit.client.0.mira043.stderr:+ wget -O test_rbd.py https://ceph.com/git/?p=ceph.git;a=blob_plain;hb=firefly;f=src/test/pybind/test_rbd.py 2014-07-01 16:19:38,865.865 INFO:teuthology.task.workunit.client.0.mira043.stderr:--2014-07-01 07:19:38-- https://ceph.com/git/?p=ceph.git;a=blob_plain;hb=firefly;f=src/test/pybind/test_rbd.py 2014-07-01 16:19:38,868.868 INFO:teuthology.task.workunit.client.0.mira043.stderr:Resolving ceph.com (ceph.com)... 208.113.241.137, 2607:f298:4:147::b05:fe2a 2014-07-01 16:19:38,869.869 INFO:teuthology.task.workunit.client.0.mira043.stderr:Connecting to ceph.com (ceph.com)|208.113.241.137|:443... connected. 2014-07-01 16:19:38,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: finishing write tid 1 to mira0435601-190 2014-07-01 16:19:38,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: finishing write tid 3 to mira0435601-190 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: finishing write tid 4 to mira0435601-190 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 115 (ObjNum 507 snap 0 seq_num 507) dirty exists 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: done (10 left) 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:574: done (9 left) 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:575: done (8 left) 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: done (7 left) 2014-07-01 16:19:38,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: done (6 left) 2014-07-01 16:19:39,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:578: done (5 left) 2014-07-01 16:19:39,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: done (4 left) 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:580: done (3 left) 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: done (2 left) 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:584: read oid 207 snap -1 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: finishing write tid 1 to mira0435601-3 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: finishing write tid 1 to mira0435601-124 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: finishing write tid 3 to mira0435601-124 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 154 current snap is 0 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: seq_num 539 ranges {1028096=696320,1724416=761856,2486272=40960} 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: writing mira0435601-154 from 1028096 to 1724416 tid 1 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: writing mira0435601-154 from 1724416 to 2486272 tid 2 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: writing mira0435601-154 from 2486272 to 2527232 tid 3 2014-07-01 16:19:39,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 20 current snap is 0 2014-07-01 16:19:39,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: seq_num 540 ranges {1867776=569344,2437120=589824,3026944=212992} 2014-07-01 16:19:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: writing mira0435601-20 from 1867776 to 2437120 tid 1 2014-07-01 16:19:39,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: writing mira0435601-20 from 2437120 to 3026944 tid 2 2014-07-01 16:19:39,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: writing mira0435601-20 from 3026944 to 3239936 tid 3 2014-07-01 16:19:39,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:19:39,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: seq_num 541 ranges {2514944=647168,3162112=401408} 2014-07-01 16:19:39,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: writing mira0435601-147 from 2514944 to 3162112 tid 1 2014-07-01 16:19:39,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: writing mira0435601-147 from 3162112 to 3563520 tid 2 2014-07-01 16:19:39,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: read oid 49 snap -1 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: finishing write tid 3 to mira0435601-3 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: finishing write tid 4 to mira0435601-3 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 3 v 83 (ObjNum 537 snap 0 seq_num 537) dirty exists 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: finishing write tid 4 to mira0435601-124 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 95 (ObjNum 538 snap 0 seq_num 538) dirty exists 2014-07-01 16:19:39,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:584: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:19:39,304.304 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 213 16 937 921 17.2922 4 1.64443 3.6636 2014-07-01 16:19:39,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: expect (ObjNum 48 snap 0 seq_num 48) 2014-07-01 16:19:40,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 214 16 945 929 17.3609 32 2.72 3.65197 2014-07-01 16:19:40,335.335 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:40,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: finishing write tid 1 to mira0435601-154 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: finishing write tid 2 to mira0435601-154 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: finishing write tid 3 to mira0435601-154 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 1 to mira0435601-20 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: finishing write tid 5 to mira0435601-154 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: finishing write tid 6 to mira0435601-154 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 128 (ObjNum 539 snap 0 seq_num 539) dirty exists 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 2 to mira0435601-20 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 3 to mira0435601-20 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 5 to mira0435601-20 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 6 to mira0435601-20 2014-07-01 16:19:40,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 110 (ObjNum 540 snap 0 seq_num 540) dirty exists 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: done (6 left) 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: done (5 left) 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:584: done (4 left) 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: done (3 left) 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: done (2 left) 2014-07-01 16:19:40,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: done (1 left) 2014-07-01 16:19:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:589: delete oid 263 current snap is 0 2014-07-01 16:19:41,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 1 to mira0435601-147 2014-07-01 16:19:41,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 2 to mira0435601-147 2014-07-01 16:19:41,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 4 to mira0435601-147 2014-07-01 16:19:41,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 5 to mira0435601-147 2014-07-01 16:19:41,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 73 (ObjNum 541 snap 0 seq_num 541) dirty exists 2014-07-01 16:19:41,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 215 16 959 943 17.5406 56 1.70978 3.64063 2014-07-01 16:19:41,310.310 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:41,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: done (1 left) 2014-07-01 16:19:41,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:589: done (0 left) 2014-07-01 16:19:41,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:590: read oid 395 snap -1 2014-07-01 16:19:41,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: read oid 339 snap -1 2014-07-01 16:19:41,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: read oid 418 snap -1 2014-07-01 16:19:41,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: read oid 212 snap -1 2014-07-01 16:19:41,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 188 current snap is 0 2014-07-01 16:19:41,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: seq_num 543 ranges {1888256=790528} 2014-07-01 16:19:41,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: writing mira0435601-188 from 1888256 to 2678784 tid 1 2014-07-01 16:19:41,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 63 current snap is 0 2014-07-01 16:19:41,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: seq_num 544 ranges {327680=417792,745472=573440,1318912=573440,1892352=458752} 2014-07-01 16:19:41,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: writing mira0435601-63 from 327680 to 745472 tid 1 2014-07-01 16:19:41,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: writing mira0435601-63 from 745472 to 1318912 tid 2 2014-07-01 16:19:41,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: writing mira0435601-63 from 1318912 to 1892352 tid 3 2014-07-01 16:19:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: writing mira0435601-63 from 1892352 to 2351104 tid 4 2014-07-01 16:19:41,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: read oid 243 snap -1 2014-07-01 16:19:41,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:590: expect (ObjNum 394 snap 0 seq_num 394) 2014-07-01 16:19:41,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: expect (ObjNum 338 snap 0 seq_num 338) 2014-07-01 16:19:42,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: expect (ObjNum 417 snap 0 seq_num 417) 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: finishing write tid 1 to mira0435601-188 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: finishing write tid 3 to mira0435601-188 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: finishing write tid 4 to mira0435601-188 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 76 (ObjNum 543 snap 0 seq_num 543) dirty exists 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 1 to mira0435601-63 2014-07-01 16:19:42,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: expect (ObjNum 211 snap 0 seq_num 211) 2014-07-01 16:19:42,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 216 16 973 957 17.7186 56 0.528241 3.59898 2014-07-01 16:19:42,366.366 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr:HTTP request sent, awaiting response... 200 OK 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr:Length: unspecified [text/x-python] 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr:Saving to: `test_rbd.py' 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr: 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr: 0K .......... .......... ........ 12.9M=0.002s 2014-07-01 16:19:42,367.367 INFO:teuthology.task.workunit.client.0.mira043.stderr: 2014-07-01 16:19:42,368.368 INFO:teuthology.task.workunit.client.0.mira043.stderr:2014-07-01 07:19:42 (12.9 MB/s) - `test_rbd.py' saved [29195] 2014-07-01 16:19:42,368.368 INFO:teuthology.task.workunit.client.0.mira043.stderr: 2014-07-01 16:19:42,368.368 INFO:teuthology.task.workunit.client.0.mira043.stderr:+ nosetests -v test_rbd 2014-07-01 16:19:43,053.053 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'5 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 127/3380 objects degraded (3.757%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 127, u'num_pgs': 144, u'data_bytes': 4515719004, u'degraded_total': 3380, u'bytes_used': 10906992640, u'op_per_sec': 0, u'version': 166, u'pgs_by_state': [{u'count': 139, u'state_name': u'active+clean'}, {u'count': 5, u'state_name': u'active+recovering'}], u'read_bytes_sec': 469, u'degraded_ratio': u'3.757', u'bytes_avail': 5987392053248}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 73, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:43,053.053 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:43,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 217 16 973 957 17.6369 0 - 3.59898 2014-07-01 16:19:43,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 2 to mira0435601-63 2014-07-01 16:19:43,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 3 to mira0435601-63 2014-07-01 16:19:43,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 4 to mira0435601-63 2014-07-01 16:19:43,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 6 to mira0435601-63 2014-07-01 16:19:43,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: expect (ObjNum 242 snap 0 seq_num 242) 2014-07-01 16:19:44,060.060 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_flatten_basic ... SKIP 2014-07-01 16:19:44,060.060 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_flatten_drops_cache ... SKIP 2014-07-01 16:19:44,062.062 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_flatten_errors ... SKIP 2014-07-01 16:19:44,062.062 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_flatten_larger_order ... SKIP 2014-07-01 16:19:44,063.063 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_flatten_smaller_order ... SKIP 2014-07-01 16:19:44,064.064 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_list_children ... SKIP 2014-07-01 16:19:44,065.065 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_read ... SKIP 2014-07-01 16:19:44,133.133 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:19:44,133.133 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_resize_io ... SKIP 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_resize_stat ... SKIP 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_stat ... SKIP 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_unprotect_with_children ... SKIP 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_unprotected ... SKIP 2014-07-01 16:19:44,228.228 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestClone.test_write ... SKIP 2014-07-01 16:19:44,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 218 16 975 959 17.5927 4 2.33159 3.5963 2014-07-01 16:19:44,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 7 to mira0435601-63 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 78 (ObjNum 544 snap 0 seq_num 544) dirty exists 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:590: done (6 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: done (5 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: done (4 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: done (3 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: done (2 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: done (1 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: done (0 left) 2014-07-01 16:19:44,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: read oid 119 snap -1 2014-07-01 16:19:44,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: read oid 17 snap -1 2014-07-01 16:19:44,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 156 current snap is 0 2014-07-01 16:19:44,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: seq_num 545 ranges {1998848=688128,2686976=557056,3244032=647168} 2014-07-01 16:19:44,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: writing mira0435601-156 from 1998848 to 2686976 tid 1 2014-07-01 16:19:44,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: writing mira0435601-156 from 2686976 to 3244032 tid 2 2014-07-01 16:19:44,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: writing mira0435601-156 from 3244032 to 3891200 tid 3 2014-07-01 16:19:44,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: read oid 172 snap -1 2014-07-01 16:19:44,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: expect (ObjNum 118 snap 0 seq_num 118) 2014-07-01 16:19:44,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: expect (ObjNum 16 snap 0 seq_num 16) 2014-07-01 16:19:44,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: done (3 left) 2014-07-01 16:19:44,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: done (2 left) 2014-07-01 16:19:44,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: read oid 225 snap -1 2014-07-01 16:19:44,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 314 current snap is 0 2014-07-01 16:19:44,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: seq_num 546 ranges {1961984=208896} 2014-07-01 16:19:44,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: writing mira0435601-314 from 1961984 to 2170880 tid 1 2014-07-01 16:19:44,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:603: delete oid 124 current snap is 0 2014-07-01 16:19:44,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: expect (ObjNum 171 snap 0 seq_num 171) 2014-07-01 16:19:44,811.811 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:45,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 219 16 982 966 17.6402 28 3.80272 3.59699 2014-07-01 16:19:46,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: finishing write tid 1 to mira0435601-156 2014-07-01 16:19:46,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: finishing write tid 2 to mira0435601-156 2014-07-01 16:19:46,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: finishing write tid 3 to mira0435601-156 2014-07-01 16:19:46,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: finishing write tid 5 to mira0435601-156 2014-07-01 16:19:46,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: finishing write tid 6 to mira0435601-156 2014-07-01 16:19:46,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 93 (ObjNum 545 snap 0 seq_num 545) dirty exists 2014-07-01 16:19:46,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: expect (ObjNum 224 snap 0 seq_num 224) 2014-07-01 16:19:46,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:19:46.219171min lat: 0.067553 max lat: 48.8206 avg lat: 3.59699 2014-07-01 16:19:46,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:19:46,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 220 16 982 966 17.56 0 - 3.59699 2014-07-01 16:19:47,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 221 16 999 983 17.7882 34 0.192851 3.58372 2014-07-01 16:19:48,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 1 to mira0435601-314 2014-07-01 16:19:48,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 3 to mira0435601-314 2014-07-01 16:19:48,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 4 to mira0435601-314 2014-07-01 16:19:48,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 112 (ObjNum 546 snap 0 seq_num 546) dirty exists 2014-07-01 16:19:48,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: done (4 left) 2014-07-01 16:19:48,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: done (3 left) 2014-07-01 16:19:48,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: done (2 left) 2014-07-01 16:19:48,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: done (1 left) 2014-07-01 16:19:48,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:603: done (0 left) 2014-07-01 16:19:48,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 75 current snap is 0 2014-07-01 16:19:48,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: seq_num 548 ranges {2334720=753664,3088384=798720,3887104=466944,4354048=339968} 2014-07-01 16:19:48,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0435601-75 from 2334720 to 3088384 tid 1 2014-07-01 16:19:48,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0435601-75 from 3088384 to 3887104 tid 2 2014-07-01 16:19:48,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0435601-75 from 3887104 to 4354048 tid 3 2014-07-01 16:19:48,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0435601-75 from 4354048 to 4694016 tid 4 2014-07-01 16:19:48,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: read oid 244 snap -1 2014-07-01 16:19:48,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: read oid 291 snap -1 2014-07-01 16:19:48,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:19:48,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: seq_num 549 ranges {368640=782336,1150976=401408} 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: writing mira0435601-280 from 368640 to 1150976 tid 1 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: writing mira0435601-280 from 1150976 to 1552384 tid 2 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 335 current snap is 0 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: seq_num 550 ranges {2363392=454656,2818048=401408} 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: writing mira0435601-335 from 2363392 to 2818048 tid 1 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: writing mira0435601-335 from 2818048 to 3219456 tid 2 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 340 current snap is 0 2014-07-01 16:19:48,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: seq_num 551 ranges {1523712=729088} 2014-07-01 16:19:48,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: writing mira0435601-340 from 1523712 to 2252800 tid 1 2014-07-01 16:19:48,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 300 current snap is 0 2014-07-01 16:19:48,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: seq_num 552 ranges {1630208=602112} 2014-07-01 16:19:48,305.305 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 222 16 999 983 17.7081 0 - 3.58372 2014-07-01 16:19:48,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: writing mira0435601-300 from 1630208 to 2232320 tid 1 2014-07-01 16:19:48,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:19:48,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: seq_num 553 ranges {147456=626688,774144=761856,1536000=442368} 2014-07-01 16:19:48,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: writing mira0435601-445 from 147456 to 774144 tid 1 2014-07-01 16:19:48,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: writing mira0435601-445 from 774144 to 1536000 tid 2 2014-07-01 16:19:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: writing mira0435601-445 from 1536000 to 1978368 tid 3 2014-07-01 16:19:48,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 299 current snap is 0 2014-07-01 16:19:48,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: seq_num 554 ranges {1757184=499712,2256896=593920,2850816=610304} 2014-07-01 16:19:48,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: writing mira0435601-299 from 1757184 to 2256896 tid 1 2014-07-01 16:19:48,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: writing mira0435601-299 from 2256896 to 2850816 tid 2 2014-07-01 16:19:48,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: writing mira0435601-299 from 2850816 to 3461120 tid 3 2014-07-01 16:19:48,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 92 current snap is 0 2014-07-01 16:19:48,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: seq_num 555 ranges {1716224=20480} 2014-07-01 16:19:48,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: writing mira0435601-92 from 1716224 to 1736704 tid 1 2014-07-01 16:19:48,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: read oid 199 snap -1 2014-07-01 16:19:48,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: expect (ObjNum 243 snap 0 seq_num 243) 2014-07-01 16:19:48,711.711 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:48,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: expect (ObjNum 290 snap 0 seq_num 290) 2014-07-01 16:19:49,244.244 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:49,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 223 16 1007 991 17.7722 16 2.5901 3.5754 2014-07-01 16:19:50,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: finishing write tid 1 to mira0435601-92 2014-07-01 16:19:50,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: finishing write tid 3 to mira0435601-92 2014-07-01 16:19:50,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: finishing write tid 4 to mira0435601-92 2014-07-01 16:19:50,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 101 (ObjNum 555 snap 0 seq_num 555) dirty exists 2014-07-01 16:19:50,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: expect (ObjNum 198 snap 0 seq_num 198) 2014-07-01 16:19:50,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 1 to mira0435601-75 2014-07-01 16:19:50,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 2 to mira0435601-75 2014-07-01 16:19:50,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 3 to mira0435601-75 2014-07-01 16:19:50,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 4 to mira0435601-75 2014-07-01 16:19:50,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 6 to mira0435601-75 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 1 to mira0435601-280 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 2 to mira0435601-280 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 4 to mira0435601-280 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 7 to mira0435601-75 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 133 (ObjNum 548 snap 0 seq_num 548) dirty exists 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 5 to mira0435601-280 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 136 (ObjNum 549 snap 0 seq_num 549) dirty exists 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: done (10 left) 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: done (9 left) 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: done (8 left) 2014-07-01 16:19:50,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: done (7 left) 2014-07-01 16:19:50,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: done (6 left) 2014-07-01 16:19:50,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: done (5 left) 2014-07-01 16:19:50,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 259 current snap is 0 2014-07-01 16:19:50,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: seq_num 556 ranges {503808=446464,950272=602112,1552384=200704} 2014-07-01 16:19:50,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: writing mira0435601-259 from 503808 to 950272 tid 1 2014-07-01 16:19:50,279.279 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:50,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: writing mira0435601-259 from 950272 to 1552384 tid 2 2014-07-01 16:19:50,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: writing mira0435601-259 from 1552384 to 1753088 tid 3 2014-07-01 16:19:50,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:616: read oid 101 snap -1 2014-07-01 16:19:50,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 16:19:50,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: seq_num 557 ranges {204800=720896,925696=401408} 2014-07-01 16:19:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: writing mira0435601-208 from 204800 to 925696 tid 1 2014-07-01 16:19:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: writing mira0435601-208 from 925696 to 1327104 tid 2 2014-07-01 16:19:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:618: read oid 319 snap -1 2014-07-01 16:19:50,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:616: expect (ObjNum 100 snap 0 seq_num 100) 2014-07-01 16:19:50,401.401 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 224 16 1018 1002 17.8892 44 3.15436 3.5602 2014-07-01 16:19:50,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:618: expect (ObjNum 318 snap 0 seq_num 318) 2014-07-01 16:19:50,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:616: done (8 left) 2014-07-01 16:19:50,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:618: done (7 left) 2014-07-01 16:19:50,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:619: delete oid 431 current snap is 0 2014-07-01 16:19:51,307.307 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 225 16 1019 1003 17.8275 4 1.07065 3.55771 2014-07-01 16:19:51,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: finishing write tid 1 to mira0435601-299 2014-07-01 16:19:51,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: finishing write tid 2 to mira0435601-299 2014-07-01 16:19:51,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: finishing write tid 3 to mira0435601-299 2014-07-01 16:19:51,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: finishing write tid 1 to mira0435601-300 2014-07-01 16:19:51,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: finishing write tid 5 to mira0435601-299 2014-07-01 16:19:51,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: finishing write tid 3 to mira0435601-300 2014-07-01 16:19:51,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: finishing write tid 1 to mira0435601-335 2014-07-01 16:19:51,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: finishing write tid 2 to mira0435601-335 2014-07-01 16:19:51,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: finishing write tid 4 to mira0435601-335 2014-07-01 16:19:51,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: finishing write tid 1 to mira0435601-340 2014-07-01 16:19:51,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: finishing write tid 3 to mira0435601-340 2014-07-01 16:19:51,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: finishing write tid 1 to mira0435601-445 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: finishing write tid 2 to mira0435601-445 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: finishing write tid 3 to mira0435601-445 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: finishing write tid 5 to mira0435601-445 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: finishing write tid 4 to mira0435601-300 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 98 (ObjNum 552 snap 0 seq_num 552) dirty exists 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: finishing write tid 5 to mira0435601-335 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 86 (ObjNum 550 snap 0 seq_num 550) dirty exists 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: finishing write tid 4 to mira0435601-340 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 88 (ObjNum 551 snap 0 seq_num 551) dirty exists 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: finishing write tid 6 to mira0435601-445 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 92 (ObjNum 553 snap 0 seq_num 553) dirty exists 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: finishing write tid 6 to mira0435601-299 2014-07-01 16:19:51,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 119 (ObjNum 554 snap 0 seq_num 554) dirty exists 2014-07-01 16:19:51,884.884 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'3 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 74/3478 objects degraded (2.128%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 5563477, u'degraded_objects': 74, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 4733528443, u'degraded_total': 3478, u'bytes_used': 11374182400, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 29906066, u'op_per_sec': 13, u'version': 172, u'pgs_by_state': [{u'count': 141, u'state_name': u'active+clean'}, {u'count': 3, u'state_name': u'active+recovering'}], u'read_bytes_sec': 831940, u'degraded_ratio': u'2.128', u'bytes_avail': 5986924863488}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 73, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:19:51,884.884 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: done (7 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: done (6 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: done (5 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: done (4 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: done (3 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:619: done (2 left) 2014-07-01 16:19:52,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 243 current snap is 0 2014-07-01 16:19:52,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: seq_num 559 ranges {1986560=667648} 2014-07-01 16:19:52,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: writing mira0435601-243 from 1986560 to 2654208 tid 1 2014-07-01 16:19:52,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: read oid 35 snap -1 2014-07-01 16:19:52,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 16:19:52,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: seq_num 560 ranges {716800=540672} 2014-07-01 16:19:52,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: writing mira0435601-418 from 716800 to 1257472 tid 1 2014-07-01 16:19:52,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:623: read oid 411 snap -1 2014-07-01 16:19:52,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: finishing write tid 1 to mira0435601-259 2014-07-01 16:19:52,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 1 to mira0435601-208 2014-07-01 16:19:52,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: finishing write tid 2 to mira0435601-259 2014-07-01 16:19:52,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 2 to mira0435601-208 2014-07-01 16:19:52,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: finishing write tid 3 to mira0435601-259 2014-07-01 16:19:52,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 4 to mira0435601-208 2014-07-01 16:19:52,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: finishing write tid 5 to mira0435601-259 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: finishing write tid 6 to mira0435601-259 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 123 (ObjNum 556 snap 0 seq_num 556) dirty exists 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 5 to mira0435601-208 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 148 (ObjNum 557 snap 0 seq_num 557) dirty exists 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: done (5 left) 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: done (4 left) 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: read oid 337 snap -1 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 42 current snap is 0 2014-07-01 16:19:52,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: seq_num 561 ranges {2138112=413696} 2014-07-01 16:19:52,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: writing mira0435601-42 from 2138112 to 2551808 tid 1 2014-07-01 16:19:52,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: read oid 331 snap -1 2014-07-01 16:19:52,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: read oid 305 snap -1 2014-07-01 16:19:52,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: expect (ObjNum 34 snap 0 seq_num 34) 2014-07-01 16:19:52,307.307 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 226 16 1038 1022 18.0848 76 2.65071 3.53071 2014-07-01 16:19:53,578.578 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:19:53,579.579 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:53,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: expect (ObjNum 330 snap 0 seq_num 330) 2014-07-01 16:19:53,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: expect (ObjNum 336 snap 0 seq_num 336) 2014-07-01 16:19:53,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 227 16 1045 1029 18.1285 28 0.799535 3.51448 2014-07-01 16:19:54,283.283 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:54,306.306 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 228 16 1054 1038 18.2068 36 1.58547 3.50075 2014-07-01 16:19:55,327.327 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 229 16 1059 1043 18.2147 20 1.23887 3.49115 2014-07-01 16:19:55,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: expect (ObjNum 304 snap 0 seq_num 304) 2014-07-01 16:19:56,308.308 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 230 16 1059 1043 18.1355 0 - 3.49115 2014-07-01 16:19:57,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: finishing write tid 1 to mira0435601-42 2014-07-01 16:19:57,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: finishing write tid 3 to mira0435601-42 2014-07-01 16:19:57,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: finishing write tid 4 to mira0435601-42 2014-07-01 16:19:57,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 94 (ObjNum 561 snap 0 seq_num 561) dirty exists 2014-07-01 16:19:57,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:623: expect (ObjNum 410 snap 0 seq_num 410) 2014-07-01 16:19:57,310.310 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 231 16 1075 1059 18.334 32 1.56631 3.47975 2014-07-01 16:19:58,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 232 16 1077 1061 18.2894 8 0.854943 3.47479 2014-07-01 16:19:58,314.314 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: finishing write tid 1 to mira0435601-243 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: finishing write tid 3 to mira0435601-243 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: finishing write tid 4 to mira0435601-243 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 117 (ObjNum 559 snap 0 seq_num 559) dirty exists 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: finishing write tid 1 to mira0435601-418 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: finishing write tid 3 to mira0435601-418 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: finishing write tid 4 to mira0435601-418 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 100 (ObjNum 560 snap 0 seq_num 560) dirty exists 2014-07-01 16:19:58,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: done (7 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: done (6 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: done (5 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:623: done (4 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: done (3 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: done (2 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: done (1 left) 2014-07-01 16:19:58,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: done (0 left) 2014-07-01 16:19:58,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:628: read oid 213 snap -1 2014-07-01 16:19:58,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 441 current snap is 0 2014-07-01 16:19:58,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: seq_num 562 ranges {860160=438272,1298432=798720,2097152=405504} 2014-07-01 16:19:58,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: writing mira0435601-441 from 860160 to 1298432 tid 1 2014-07-01 16:19:58,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: writing mira0435601-441 from 1298432 to 2097152 tid 2 2014-07-01 16:19:58,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: writing mira0435601-441 from 2097152 to 2502656 tid 3 2014-07-01 16:19:58,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:630: read oid 445 snap -1 2014-07-01 16:19:58,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:628: expect (ObjNum 212 snap 0 seq_num 212) 2014-07-01 16:19:58,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:630: expect (ObjNum 553 snap 0 seq_num 553) 2014-07-01 16:19:58,886.886 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:19:59,307.307 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 233 16 1081 1065 18.2796 16 2.0381 3.46925 2014-07-01 16:19:59,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:628: done (2 left) 2014-07-01 16:19:59,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:630: done (1 left) 2014-07-01 16:19:59,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 16:19:59,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: seq_num 563 ranges {2056192=712704,2768896=737280,3506176=65536} 2014-07-01 16:19:59,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: writing mira0435601-250 from 2056192 to 2768896 tid 1 2014-07-01 16:19:59,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: writing mira0435601-250 from 2768896 to 3506176 tid 2 2014-07-01 16:19:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: writing mira0435601-250 from 3506176 to 3571712 tid 3 2014-07-01 16:19:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 16:19:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: seq_num 564 ranges {1654784=790528,2445312=401408} 2014-07-01 16:19:59,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: writing mira0435601-229 from 1654784 to 2445312 tid 1 2014-07-01 16:19:59,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: writing mira0435601-229 from 2445312 to 2846720 tid 2 2014-07-01 16:19:59,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 192 current snap is 0 2014-07-01 16:19:59,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: seq_num 565 ranges {1179648=659456,1839104=401408} 2014-07-01 16:19:59,933.933 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:19:59,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: writing mira0435601-192 from 1179648 to 1839104 tid 1 2014-07-01 16:19:59,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: writing mira0435601-192 from 1839104 to 2240512 tid 2 2014-07-01 16:19:59,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 268 current snap is 0 2014-07-01 16:19:59,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: seq_num 566 ranges {1675264=532480,2207744=401408} 2014-07-01 16:19:59,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: writing mira0435601-268 from 1675264 to 2207744 tid 1 2014-07-01 16:19:59,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: writing mira0435601-268 from 2207744 to 2609152 tid 2 2014-07-01 16:19:59,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:635: delete oid 95 current snap is 0 2014-07-01 16:20:00,307.307 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 234 16 1083 1067 18.2356 8 3.36629 3.46909 2014-07-01 16:20:00,654.654 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:19:54.527803', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887640660, u'health': u'HEALTH_OK', u'kb_used': 9140392, u'store_stats': {u'bytes_total': 11732329, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 25/3631 objects degraded (0.689%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 14357441, u'degraded_objects': 25, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 5064399451, u'degraded_total': 3631, u'bytes_used': 12416258048, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 39178709, u'op_per_sec': 13, u'version': 178, u'pgs_by_state': [{u'count': 143, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'active+recovering'}], u'read_bytes_sec': 688778, u'degraded_ratio': u'0.689', u'bytes_avail': 5985882787840}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 73, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:00,654.654 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:01,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 235 16 1086 1070 18.2091 12 3.66409 3.47097 2014-07-01 16:20:01,421.421 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:20:01,421.421 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:02,251.251 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:02,315.315 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 236 16 1089 1073 18.1828 12 0.11798 3.46426 2014-07-01 16:20:03,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 237 16 1090 1074 18.1229 4 0.745719 3.46173 2014-07-01 16:20:04,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 1 to mira0435601-250 2014-07-01 16:20:04,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 2 to mira0435601-250 2014-07-01 16:20:04,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 3 to mira0435601-250 2014-07-01 16:20:04,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 5 to mira0435601-250 2014-07-01 16:20:04,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 6 to mira0435601-250 2014-07-01 16:20:04,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 85 (ObjNum 563 snap 0 seq_num 563) dirty exists 2014-07-01 16:20:04,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: finishing write tid 1 to mira0435601-192 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: finishing write tid 2 to mira0435601-192 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: finishing write tid 4 to mira0435601-192 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: done (5 left) 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:635: done (4 left) 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 265 current snap is 0 2014-07-01 16:20:04,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: seq_num 568 ranges {897024=430080,1327104=790528,2117632=704512,2822144=110592} 2014-07-01 16:20:04,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 238 16 1103 1087 18.2652 52 3.36865 3.49175 2014-07-01 16:20:04,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0435601-265 from 897024 to 1327104 tid 1 2014-07-01 16:20:04,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0435601-265 from 1327104 to 2117632 tid 2 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0435601-265 from 2117632 to 2822144 tid 3 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0435601-265 from 2822144 to 2932736 tid 4 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 328 current snap is 0 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: seq_num 569 ranges {4374528=352256} 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: writing mira0435601-328 from 4374528 to 4726784 tid 1 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:638: read oid 411 snap -1 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: finishing write tid 5 to mira0435601-192 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 107 (ObjNum 565 snap 0 seq_num 565) dirty exists 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 1 to mira0435601-441 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: finishing write tid 1 to mira0435601-268 2014-07-01 16:20:04,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: finishing write tid 2 to mira0435601-268 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: finishing write tid 4 to mira0435601-268 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 2 to mira0435601-441 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 3 to mira0435601-441 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 5 to mira0435601-441 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: finishing write tid 1 to mira0435601-229 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: finishing write tid 2 to mira0435601-229 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: finishing write tid 4 to mira0435601-229 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 6 to mira0435601-441 2014-07-01 16:20:04,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 105 (ObjNum 562 snap 0 seq_num 562) dirty exists 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: finishing write tid 5 to mira0435601-229 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 108 (ObjNum 564 snap 0 seq_num 564) dirty exists 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: finishing write tid 5 to mira0435601-268 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 97 (ObjNum 566 snap 0 seq_num 566) dirty exists 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: done (6 left) 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: done (5 left) 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: done (4 left) 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: done (3 left) 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 438 current snap is 0 2014-07-01 16:20:04,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: seq_num 570 ranges {81920=778240,860160=606208,1466368=196608} 2014-07-01 16:20:04,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: writing mira0435601-438 from 81920 to 860160 tid 1 2014-07-01 16:20:04,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: writing mira0435601-438 from 860160 to 1466368 tid 2 2014-07-01 16:20:04,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: writing mira0435601-438 from 1466368 to 1662976 tid 3 2014-07-01 16:20:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: read oid 344 snap -1 2014-07-01 16:20:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 477 current snap is 0 2014-07-01 16:20:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: seq_num 571 ranges {561152=651264,1212416=540672,1753088=262144} 2014-07-01 16:20:04,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: writing mira0435601-477 from 561152 to 1212416 tid 1 2014-07-01 16:20:04,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: writing mira0435601-477 from 1212416 to 1753088 tid 2 2014-07-01 16:20:04,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: writing mira0435601-477 from 1753088 to 2015232 tid 3 2014-07-01 16:20:04,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: read oid 309 snap -1 2014-07-01 16:20:04,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 227 current snap is 0 2014-07-01 16:20:04,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: seq_num 572 ranges {552960=524288,1077248=770048,1847296=32768} 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: writing mira0435601-227 from 552960 to 1077248 tid 1 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: writing mira0435601-227 from 1077248 to 1847296 tid 2 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: writing mira0435601-227 from 1847296 to 1880064 tid 3 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: seq_num 573 ranges {2031616=598016,2629632=401408} 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: writing mira0435601-211 from 2031616 to 2629632 tid 1 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: writing mira0435601-211 from 2629632 to 3031040 tid 2 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 215 current snap is 0 2014-07-01 16:20:04,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: seq_num 574 ranges {1323008=471040,1794048=401408} 2014-07-01 16:20:04,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: writing mira0435601-215 from 1323008 to 1794048 tid 1 2014-07-01 16:20:04,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: writing mira0435601-215 from 1794048 to 2195456 tid 2 2014-07-01 16:20:04,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:646: delete oid 288 current snap is 0 2014-07-01 16:20:04,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:638: expect (ObjNum 410 snap 0 seq_num 410) 2014-07-01 16:20:04,975.975 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_copy ... ok 2014-07-01 16:20:05,308.308 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 239 16 1120 1104 18.4733 68 0.71723 3.45777 2014-07-01 16:20:06,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: expect (ObjNum 308 snap 0 seq_num 308) 2014-07-01 16:20:06,121.121 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:06,308.308 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:20:06.222280min lat: 0.067553 max lat: 48.8206 avg lat: 3.45153 2014-07-01 16:20:06,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:20:06,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 240 16 1123 1107 18.4463 12 1.55786 3.45153 2014-07-01 16:20:06,854.854 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:06,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: expect (ObjNum 343 snap 0 seq_num 343) 2014-07-01 16:20:07,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 1 to mira0435601-265 2014-07-01 16:20:07,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 2 to mira0435601-265 2014-07-01 16:20:07,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 3 to mira0435601-265 2014-07-01 16:20:07,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 4 to mira0435601-265 2014-07-01 16:20:07,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 6 to mira0435601-265 2014-07-01 16:20:07,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: finishing write tid 1 to mira0435601-328 2014-07-01 16:20:07,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: finishing write tid 1 to mira0435601-438 2014-07-01 16:20:07,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: finishing write tid 3 to mira0435601-328 2014-07-01 16:20:07,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: finishing write tid 2 to mira0435601-438 2014-07-01 16:20:07,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: finishing write tid 3 to mira0435601-438 2014-07-01 16:20:07,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: finishing write tid 5 to mira0435601-438 2014-07-01 16:20:07,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: finishing write tid 1 to mira0435601-227 2014-07-01 16:20:07,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:638: done (10 left) 2014-07-01 16:20:07,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: done (9 left) 2014-07-01 16:20:07,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: done (8 left) 2014-07-01 16:20:07,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:646: done (7 left) 2014-07-01 16:20:07,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 371 current snap is 0 2014-07-01 16:20:07,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: seq_num 576 ranges {1605632=770048,2375680=753664,3129344=450560} 2014-07-01 16:20:07,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 241 16 1123 1107 18.3698 0 - 3.45153 2014-07-01 16:20:07,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: writing mira0435601-371 from 1605632 to 2375680 tid 1 2014-07-01 16:20:07,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: writing mira0435601-371 from 2375680 to 3129344 tid 2 2014-07-01 16:20:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: writing mira0435601-371 from 3129344 to 3579904 tid 3 2014-07-01 16:20:07,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 489 current snap is 0 2014-07-01 16:20:07,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: seq_num 577 ranges {692224=798720} 2014-07-01 16:20:07,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: writing mira0435601-489 from 692224 to 1490944 tid 1 2014-07-01 16:20:07,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 373 current snap is 0 2014-07-01 16:20:07,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: seq_num 578 ranges {303104=163840} 2014-07-01 16:20:07,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: writing mira0435601-373 from 303104 to 466944 tid 1 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: read oid 132 snap -1 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 1 to mira0435601-477 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: read oid 470 snap -1 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 2 to mira0435601-477 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 3 to mira0435601-477 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 5 to mira0435601-477 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: finishing write tid 2 to mira0435601-227 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: finishing write tid 3 to mira0435601-227 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: finishing write tid 5 to mira0435601-227 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: finishing write tid 1 to mira0435601-215 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: finishing write tid 2 to mira0435601-215 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: finishing write tid 1 to mira0435601-211 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: finishing write tid 4 to mira0435601-215 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: finishing write tid 2 to mira0435601-211 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: finishing write tid 4 to mira0435601-211 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: read oid 345 snap -1 2014-07-01 16:20:07,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 7 to mira0435601-265 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 265 v 98 (ObjNum 568 snap 0 seq_num 568) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: finishing write tid 4 to mira0435601-328 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 114 (ObjNum 569 snap 0 seq_num 569) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: finishing write tid 6 to mira0435601-438 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 102 (ObjNum 570 snap 0 seq_num 570) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 6 to mira0435601-477 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 82 (ObjNum 571 snap 0 seq_num 571) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: finishing write tid 6 to mira0435601-227 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 110 (ObjNum 572 snap 0 seq_num 572) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: finishing write tid 5 to mira0435601-211 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 70 (ObjNum 573 snap 0 seq_num 573) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: finishing write tid 5 to mira0435601-215 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 103 (ObjNum 574 snap 0 seq_num 574) dirty exists 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: done (12 left) 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: done (11 left) 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: done (10 left) 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: done (9 left) 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: done (8 left) 2014-07-01 16:20:07,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: done (7 left) 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: done (6 left) 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:653: read oid 262 snap -1 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: seq_num 579 ranges {1462272=589824,2052096=749568,2801664=53248} 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: writing mira0435601-9 from 1462272 to 2052096 tid 1 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: writing mira0435601-9 from 2052096 to 2801664 tid 2 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: writing mira0435601-9 from 2801664 to 2854912 tid 3 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: read oid 439 snap -1 2014-07-01 16:20:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 16:20:07,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: expect (ObjNum 469 snap 0 seq_num 469) 2014-07-01 16:20:07,719.719 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:08,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 242 16 1134 1118 18.4756 22 0.145837 3.44518 2014-07-01 16:20:08,431.431 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:19:54.527803', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887640660, u'health': u'HEALTH_OK', u'kb_used': 9140392, u'store_stats': {u'bytes_total': 11732329, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:20:01.657127', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887639380, u'health': u'HEALTH_OK', u'kb_used': 9141672, u'store_stats': {u'bytes_total': 11740564, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 12/3742 objects degraded (0.321%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 36665561, u'degraded_objects': 12, u'num_pgs': 144, u'recovering_keys_per_sec': 0, u'data_bytes': 5315947099, u'degraded_total': 3742, u'bytes_used': 13227794432, u'recovering_objects_per_sec': 8, u'write_bytes_sec': 104605861, u'op_per_sec': 67, u'version': 185, u'pgs_by_state': [{u'count': 143, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'active+recovering'}], u'read_bytes_sec': 7567046, u'degraded_ratio': u'0.321', u'bytes_avail': 5985071251456}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 74, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:08,431.431 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:09,236.236 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:20:09,236.236 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:09,310.310 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 243 16 1151 1135 18.6794 68 0.318669 3.42267 2014-07-01 16:20:09,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:653: expect (ObjNum 261 snap 0 seq_num 261) 2014-07-01 16:20:09,950.950 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:10,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 244 16 1158 1142 18.7176 28 1.03157 3.40655 2014-07-01 16:20:10,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: expect (ObjNum 438 snap 0 seq_num 438) 2014-07-01 16:20:11,310.310 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 245 16 1160 1144 18.6738 8 0.785791 3.40221 2014-07-01 16:20:11,691.691 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_create_snap ... ok 2014-07-01 16:20:11,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: expect (ObjNum 344 snap 0 seq_num 344) 2014-07-01 16:20:12,309.309 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 246 16 1164 1148 18.6629 16 0.63675 3.39391 2014-07-01 16:20:13,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 247 16 1173 1157 18.7331 36 3.72892 3.39858 2014-07-01 16:20:13,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: finishing write tid 1 to mira0435601-373 2014-07-01 16:20:13,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: finishing write tid 1 to mira0435601-371 2014-07-01 16:20:13,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: finishing write tid 2 to mira0435601-371 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: finishing write tid 1 to mira0435601-489 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: finishing write tid 3 to mira0435601-371 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: finishing write tid 3 to mira0435601-373 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: finishing write tid 3 to mira0435601-489 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: finishing write tid 5 to mira0435601-371 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: finishing write tid 1 to mira0435601-9 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: finishing write tid 2 to mira0435601-9 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: finishing write tid 3 to mira0435601-9 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: finishing write tid 5 to mira0435601-9 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: finishing write tid 4 to mira0435601-373 2014-07-01 16:20:13,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 116 (ObjNum 578 snap 0 seq_num 578) dirty exists 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: finishing write tid 6 to mira0435601-371 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 153 (ObjNum 576 snap 0 seq_num 576) dirty exists 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: finishing write tid 4 to mira0435601-489 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 105 (ObjNum 577 snap 0 seq_num 577) dirty exists 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: finishing write tid 6 to mira0435601-9 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 157 (ObjNum 579 snap 0 seq_num 579) dirty exists 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: done (8 left) 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: done (7 left) 2014-07-01 16:20:13,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: done (6 left) 2014-07-01 16:20:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: done (5 left) 2014-07-01 16:20:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: done (4 left) 2014-07-01 16:20:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: done (3 left) 2014-07-01 16:20:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:653: done (2 left) 2014-07-01 16:20:13,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: done (1 left) 2014-07-01 16:20:13,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: done (0 left) 2014-07-01 16:20:13,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 16:20:13,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: seq_num 580 ranges {1490944=217088} 2014-07-01 16:20:13,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: writing mira0435601-434 from 1490944 to 1708032 tid 1 2014-07-01 16:20:13,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: read oid 350 snap -1 2014-07-01 16:20:13,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: read oid 137 snap -1 2014-07-01 16:20:13,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:659: delete oid 121 current snap is 0 2014-07-01 16:20:13,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: expect (ObjNum 136 snap 0 seq_num 136) 2014-07-01 16:20:13,793.793 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:13,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: expect (ObjNum 349 snap 0 seq_num 349) 2014-07-01 16:20:14,310.310 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 248 16 1173 1157 18.6576 0 - 3.39858 2014-07-01 16:20:14,361.361 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:15,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: done (3 left) 2014-07-01 16:20:15,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: done (2 left) 2014-07-01 16:20:15,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:659: done (1 left) 2014-07-01 16:20:15,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:660: read oid 300 snap -1 2014-07-01 16:20:15,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 441 current snap is 0 2014-07-01 16:20:15,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: seq_num 582 ranges {2502656=409600,2912256=401408} 2014-07-01 16:20:15,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0435601-441 from 2502656 to 2912256 tid 1 2014-07-01 16:20:15,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0435601-441 from 2912256 to 3313664 tid 2 2014-07-01 16:20:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 5 current snap is 0 2014-07-01 16:20:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: seq_num 583 ranges {811008=438272,1249280=729088,1978368=663552,2641920=212992} 2014-07-01 16:20:15,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: writing mira0435601-5 from 811008 to 1249280 tid 1 2014-07-01 16:20:15,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: writing mira0435601-5 from 1249280 to 1978368 tid 2 2014-07-01 16:20:15,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: writing mira0435601-5 from 1978368 to 2641920 tid 3 2014-07-01 16:20:15,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: writing mira0435601-5 from 2641920 to 2854912 tid 4 2014-07-01 16:20:15,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 18 current snap is 0 2014-07-01 16:20:15,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: seq_num 584 ranges {1273856=557056} 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: writing mira0435601-18 from 1273856 to 1830912 tid 1 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:664: read oid 403 snap -1 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 1 to mira0435601-434 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 3 to mira0435601-434 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 4 to mira0435601-434 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 112 (ObjNum 580 snap 0 seq_num 580) dirty exists 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: done (5 left) 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 453 current snap is 0 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: seq_num 585 ranges {90112=581632,671744=544768,1216512=610304,1826816=49152} 2014-07-01 16:20:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: writing mira0435601-453 from 90112 to 671744 tid 1 2014-07-01 16:20:15,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: writing mira0435601-453 from 671744 to 1216512 tid 2 2014-07-01 16:20:15,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: writing mira0435601-453 from 1216512 to 1826816 tid 3 2014-07-01 16:20:15,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: writing mira0435601-453 from 1826816 to 1875968 tid 4 2014-07-01 16:20:15,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 396 current snap is 0 2014-07-01 16:20:15,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: seq_num 586 ranges {446464=610304} 2014-07-01 16:20:15,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: writing mira0435601-396 from 446464 to 1056768 tid 1 2014-07-01 16:20:15,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:667: read oid 382 snap -1 2014-07-01 16:20:15,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:660: expect (ObjNum 552 snap 0 seq_num 552) 2014-07-01 16:20:15,311.311 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 249 16 1192 1176 18.8878 38 0.218496 3.38547 2014-07-01 16:20:15,392.392 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:16,311.311 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 250 16 1203 1187 18.9882 44 0.931363 3.35934 2014-07-01 16:20:16,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 1 to mira0435601-441 2014-07-01 16:20:16,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 2 to mira0435601-441 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 4 to mira0435601-441 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 5 to mira0435601-441 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 111 (ObjNum 582 snap 0 seq_num 582) dirty exists 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 1 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:660: done (7 left) 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: done (6 left) 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: delete oid 117 current snap is 0 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 2 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 3 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 4 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 6 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: finishing write tid 7 to mira0435601-5 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 128 (ObjNum 583 snap 0 seq_num 583) dirty exists 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 1 to mira0435601-453 2014-07-01 16:20:16,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: finishing write tid 1 to mira0435601-396 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: finishing write tid 3 to mira0435601-396 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: finishing write tid 4 to mira0435601-396 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 139 (ObjNum 586 snap 0 seq_num 586) dirty exists 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 2 to mira0435601-453 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: finishing write tid 1 to mira0435601-18 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: finishing write tid 3 to mira0435601-18 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: finishing write tid 4 to mira0435601-18 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 18 v 99 (ObjNum 584 snap 0 seq_num 584) dirty exists 2014-07-01 16:20:16,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:667: expect (ObjNum 381 snap 0 seq_num 381) 2014-07-01 16:20:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 3 to mira0435601-453 2014-07-01 16:20:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 4 to mira0435601-453 2014-07-01 16:20:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 6 to mira0435601-453 2014-07-01 16:20:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 7 to mira0435601-453 2014-07-01 16:20:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 113 (ObjNum 585 snap 0 seq_num 585) dirty exists 2014-07-01 16:20:17,261.261 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:19:54.527803', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887640660, u'health': u'HEALTH_OK', u'kb_used': 9140392, u'store_stats': {u'bytes_total': 11732329, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:20:01.657127', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887639380, u'health': u'HEALTH_OK', u'kb_used': 9141672, u'store_stats': {u'bytes_total': 11740564, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 144, u'data_bytes': 5540731381, u'bytes_used': 13586329600, u'op_per_sec': 48, u'version': 195, u'pgs_by_state': [{u'count': 144, u'state_name': u'active+clean'}], u'read_bytes_sec': 1212217, u'write_bytes_sec': 42037729, u'bytes_avail': 5984712716288}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 78, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:17,262.262 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:20:17,262.262 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:20:17,262.262 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 1 to 0.577148015674 2014-07-01 16:20:17,263.263 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 0.577148015674' 2014-07-01 16:20:17,310.310 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 251 16 1208 1192 18.9922 20 1.72933 3.35151 2014-07-01 16:20:18,311.311 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 252 16 1214 1198 19.0121 24 2.65318 3.3469 2014-07-01 16:20:18,435.435 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 0.577148 (8378232) 2014-07-01 16:20:19,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 253 16 1214 1198 18.9369 0 - 3.3469 2014-07-01 16:20:19,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: done (6 left) 2014-07-01 16:20:19,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: done (5 left) 2014-07-01 16:20:19,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:664: done (4 left) 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: done (3 left) 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: done (2 left) 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:667: done (1 left) 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: done (0 left) 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 16:20:19,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: seq_num 588 ranges {3031040=802816,3833856=401408} 2014-07-01 16:20:19,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: writing mira0435601-211 from 3031040 to 3833856 tid 1 2014-07-01 16:20:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: writing mira0435601-211 from 3833856 to 4235264 tid 2 2014-07-01 16:20:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:670: delete oid 213 current snap is 0 2014-07-01 16:20:19,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:670: done (1 left) 2014-07-01 16:20:19,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 44 current snap is 0 2014-07-01 16:20:19,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: seq_num 590 ranges {835584=700416,1536000=753664,2289664=483328,2772992=368640} 2014-07-01 16:20:19,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: writing mira0435601-44 from 835584 to 1536000 tid 1 2014-07-01 16:20:19,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: writing mira0435601-44 from 1536000 to 2289664 tid 2 2014-07-01 16:20:19,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: writing mira0435601-44 from 2289664 to 2772992 tid 3 2014-07-01 16:20:19,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: writing mira0435601-44 from 2772992 to 3141632 tid 4 2014-07-01 16:20:19,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 384 current snap is 0 2014-07-01 16:20:19,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: seq_num 591 ranges {1216512=622592} 2014-07-01 16:20:19,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: writing mira0435601-384 from 1216512 to 1839104 tid 1 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:673: read oid 2 snap -1 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: finishing write tid 1 to mira0435601-211 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: finishing write tid 2 to mira0435601-211 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: finishing write tid 4 to mira0435601-211 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: finishing write tid 5 to mira0435601-211 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 73 (ObjNum 588 snap 0 seq_num 588) dirty exists 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: done (3 left) 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: read oid 309 snap -1 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: read oid 167 snap -1 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:676: delete oid 196 current snap is 0 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:676: done (5 left) 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:677: read oid 386 snap -1 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 279 current snap is 0 2014-07-01 16:20:19,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: seq_num 593 ranges {180224=520192,700416=749568,1449984=454656} 2014-07-01 16:20:19,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: writing mira0435601-279 from 180224 to 700416 tid 1 2014-07-01 16:20:19,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: writing mira0435601-279 from 700416 to 1449984 tid 2 2014-07-01 16:20:19,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: writing mira0435601-279 from 1449984 to 1904640 tid 3 2014-07-01 16:20:19,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 1 to mira0435601-44 2014-07-01 16:20:19,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:673: expect (ObjNum 1 snap 0 seq_num 1) 2014-07-01 16:20:19,798.798 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_diff_iterate ... ok 2014-07-01 16:20:19,937.937 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_large_read ... ok 2014-07-01 16:20:20,311.311 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 254 16 1232 1216 19.1458 36 0.738654 3.3289 2014-07-01 16:20:21,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 255 16 1234 1218 19.1021 8 1.44323 3.32582 2014-07-01 16:20:21,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: expect (ObjNum 166 snap 0 seq_num 166) 2014-07-01 16:20:22,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 256 16 1234 1218 19.0275 0 - 3.32582 2014-07-01 16:20:22,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 2 to mira0435601-44 2014-07-01 16:20:22,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:677: expect (ObjNum 385 snap 0 seq_num 385) 2014-07-01 16:20:23,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 257 16 1247 1231 19.1557 26 3.36284 3.32342 2014-07-01 16:20:23,448.448 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:20:23,449.449 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:20:23,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: finishing write tid 1 to mira0435601-279 2014-07-01 16:20:23,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: finishing write tid 2 to mira0435601-279 2014-07-01 16:20:23,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: finishing write tid 3 to mira0435601-279 2014-07-01 16:20:23,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: finishing write tid 5 to mira0435601-279 2014-07-01 16:20:23,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: finishing write tid 6 to mira0435601-279 2014-07-01 16:20:23,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 116 (ObjNum 593 snap 0 seq_num 593) dirty exists 2014-07-01 16:20:23,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 3 to mira0435601-44 2014-07-01 16:20:23,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 4 to mira0435601-44 2014-07-01 16:20:23,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 6 to mira0435601-44 2014-07-01 16:20:23,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:673: done (6 left) 2014-07-01 16:20:23,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: done (5 left) 2014-07-01 16:20:23,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:677: done (4 left) 2014-07-01 16:20:23,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: done (3 left) 2014-07-01 16:20:23,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:679: read oid 39 snap -1 2014-07-01 16:20:23,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: read oid 276 snap -1 2014-07-01 16:20:23,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: delete oid 360 current snap is 0 2014-07-01 16:20:23,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:679: expect (ObjNum 38 snap 0 seq_num 38) 2014-07-01 16:20:23,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: expect (ObjNum 275 snap 0 seq_num 275) 2014-07-01 16:20:24,311.311 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 258 16 1254 1238 19.19 28 0.903536 3.31079 2014-07-01 16:20:24,527.527 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:20:24,549.549 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:20:25,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 259 16 1256 1240 19.1468 8 1.68162 3.30825 2014-07-01 16:20:25,572.572 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:20:25,585.585 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:20:26,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:20:26.225340min lat: 0.067553 max lat: 48.8206 avg lat: 3.30647 2014-07-01 16:20:26,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:20:26,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 260 16 1262 1246 19.1655 24 3.28568 3.30647 2014-07-01 16:20:26,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:679: done (5 left) 2014-07-01 16:20:26,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: done (4 left) 2014-07-01 16:20:26,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: done (3 left) 2014-07-01 16:20:26,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:20:26,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: seq_num 595 ranges {2854912=667648,3522560=516096,4038656=286720} 2014-07-01 16:20:26,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: writing mira0435601-9 from 2854912 to 3522560 tid 1 2014-07-01 16:20:26,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: writing mira0435601-9 from 3522560 to 4038656 tid 2 2014-07-01 16:20:26,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: writing mira0435601-9 from 4038656 to 4325376 tid 3 2014-07-01 16:20:26,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: read oid 38 snap -1 2014-07-01 16:20:26,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 289 current snap is 0 2014-07-01 16:20:26,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: seq_num 596 ranges {2273280=741376,3014656=401408} 2014-07-01 16:20:26,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: writing mira0435601-289 from 2273280 to 3014656 tid 1 2014-07-01 16:20:26,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: writing mira0435601-289 from 3014656 to 3416064 tid 2 2014-07-01 16:20:26,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:685: read oid 100 snap -1 2014-07-01 16:20:26,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: expect (ObjNum 37 snap 0 seq_num 37) 2014-07-01 16:20:26,615.615 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:20:26,628.628 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:20:26,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:685: expect (ObjNum 99 snap 0 seq_num 99) 2014-07-01 16:20:27,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 261 16 1271 1255 19.2299 36 1.92847 3.29998 2014-07-01 16:20:27,584.584 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:20:27,599.599 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:20:28,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 262 16 1285 1269 19.3702 56 0.207795 3.27712 2014-07-01 16:20:28,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: finishing write tid 7 to mira0435601-44 2014-07-01 16:20:28,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 44 v 162 (ObjNum 590 snap 0 seq_num 590) dirty exists 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: finishing write tid 1 to mira0435601-9 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: finishing write tid 2 to mira0435601-9 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: finishing write tid 3 to mira0435601-9 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: done (6 left) 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: done (5 left) 2014-07-01 16:20:28,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:685: done (4 left) 2014-07-01 16:20:28,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:686: delete oid 19 current snap is 0 2014-07-01 16:20:28,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: finishing write tid 5 to mira0435601-9 2014-07-01 16:20:28,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: expect (ObjNum 308 snap 0 seq_num 308) 2014-07-01 16:20:29,021.021 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:20:29,032.032 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:20:29,312.312 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 263 16 1285 1269 19.2966 0 - 3.27712 2014-07-01 16:20:30,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 264 16 1285 1269 19.2235 0 - 3.27712 2014-07-01 16:20:31,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 265 16 1287 1271 19.1811 2.66667 4.22005 3.27864 2014-07-01 16:20:32,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 266 16 1287 1271 19.109 0 - 3.27864 2014-07-01 16:20:33,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 267 16 1287 1271 19.0375 0 - 3.27864 2014-07-01 16:20:34,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 268 16 1287 1271 18.9664 0 - 3.27864 2014-07-01 16:20:35,157.157 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:20:35,167.167 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:20:35,168.168 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:35,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: finishing write tid 6 to mira0435601-9 2014-07-01 16:20:35,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 166 (ObjNum 595 snap 0 seq_num 595) dirty exists 2014-07-01 16:20:35,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 269 16 1289 1273 18.9257 2 7.19594 3.28489 2014-07-01 16:20:35,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: finishing write tid 1 to mira0435601-289 2014-07-01 16:20:35,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: finishing write tid 2 to mira0435601-289 2014-07-01 16:20:35,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: finishing write tid 4 to mira0435601-289 2014-07-01 16:20:35,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: finishing write tid 5 to mira0435601-289 2014-07-01 16:20:35,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 102 (ObjNum 596 snap 0 seq_num 596) dirty exists 2014-07-01 16:20:36,059.059 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:36,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: done (4 left) 2014-07-01 16:20:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: done (3 left) 2014-07-01 16:20:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: done (2 left) 2014-07-01 16:20:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:686: done (1 left) 2014-07-01 16:20:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 86 current snap is 0 2014-07-01 16:20:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: seq_num 598 ranges {471040=487424,958464=401408} 2014-07-01 16:20:36,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: writing mira0435601-86 from 471040 to 958464 tid 1 2014-07-01 16:20:36,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: writing mira0435601-86 from 958464 to 1359872 tid 2 2014-07-01 16:20:36,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: read oid 365 snap -1 2014-07-01 16:20:36,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: read oid 113 snap -1 2014-07-01 16:20:36,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:690: delete oid 245 current snap is 0 2014-07-01 16:20:36,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: expect (ObjNum 364 snap 0 seq_num 364) 2014-07-01 16:20:36,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 270 16 1304 1288 19.0778 60 0.152919 3.32019 2014-07-01 16:20:36,933.933 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:37,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 271 16 1309 1293 19.0812 20 5.9567 3.32754 2014-07-01 16:20:37,664.664 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:37,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: expect (ObjNum 112 snap 0 seq_num 112) 2014-07-01 16:20:38,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 272 16 1323 1307 19.2168 56 10.2196 3.31447 2014-07-01 16:20:38,531.531 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:38,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: finishing write tid 1 to mira0435601-86 2014-07-01 16:20:38,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: finishing write tid 2 to mira0435601-86 2014-07-01 16:20:38,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: finishing write tid 4 to mira0435601-86 2014-07-01 16:20:38,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: finishing write tid 1 to mira0435601-384 2014-07-01 16:20:38,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: finishing write tid 3 to mira0435601-384 2014-07-01 16:20:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: finishing write tid 4 to mira0435601-384 2014-07-01 16:20:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 134 (ObjNum 591 snap 0 seq_num 591) dirty exists 2014-07-01 16:20:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: finishing write tid 5 to mira0435601-86 2014-07-01 16:20:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 105 (ObjNum 598 snap 0 seq_num 598) dirty exists 2014-07-01 16:20:39,248.248 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:20:35.052877', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884362072, u'health': u'HEALTH_OK', u'kb_used': 12418980, u'store_stats': {u'bytes_total': 4488119, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1342375}}, {u'last_updated': u'2014-07-01 07:19:54.527803', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887640660, u'health': u'HEALTH_OK', u'kb_used': 9140392, u'store_stats': {u'bytes_total': 11732329, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:20:01.657127', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887639380, u'health': u'HEALTH_OK', u'kb_used': 9141672, u'store_stats': {u'bytes_total': 11740564, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'8 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 517/4071 objects degraded (12.700%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 517, u'num_pgs': 144, u'data_bytes': 6066636767, u'degraded_total': 4071, u'bytes_used': 14258114560, u'op_per_sec': 2, u'write_bytes_sec': 11120895, u'version': 214, u'pgs_by_state': [{u'count': 136, u'state_name': u'active+clean'}, {u'count': 8, u'state_name': u'active+recovering'}], u'read_bytes_sec': 339, u'degraded_ratio': u'12.700', u'bytes_avail': 5984040931328}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 89, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:39,248.248 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:39,315.315 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 273 16 1328 1312 19.2197 20 0.624474 3.30456 2014-07-01 16:20:40,004.004 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:20:40,005.005 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:40,313.313 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 274 16 1334 1318 19.2371 24 1.14497 3.29536 2014-07-01 16:20:40,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: done (4 left) 2014-07-01 16:20:40,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: done (3 left) 2014-07-01 16:20:40,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: done (2 left) 2014-07-01 16:20:40,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: done (1 left) 2014-07-01 16:20:40,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:690: done (0 left) 2014-07-01 16:20:40,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 89 current snap is 0 2014-07-01 16:20:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: seq_num 600 ranges {2297856=589824,2887680=724992,3612672=479232} 2014-07-01 16:20:40,779.779 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:40,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: writing mira0435601-89 from 2297856 to 2887680 tid 1 2014-07-01 16:20:40,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: writing mira0435601-89 from 2887680 to 3612672 tid 2 2014-07-01 16:20:40,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: writing mira0435601-89 from 3612672 to 4091904 tid 3 2014-07-01 16:20:40,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 411 current snap is 0 2014-07-01 16:20:40,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: seq_num 601 ranges {2007040=307200} 2014-07-01 16:20:40,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: writing mira0435601-411 from 2007040 to 2314240 tid 1 2014-07-01 16:20:40,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: read oid 218 snap -1 2014-07-01 16:20:40,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 16:20:40,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: seq_num 602 ranges {2236416=733184,2969600=540672,3510272=262144} 2014-07-01 16:20:40,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: writing mira0435601-336 from 2236416 to 2969600 tid 1 2014-07-01 16:20:40,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: writing mira0435601-336 from 2969600 to 3510272 tid 2 2014-07-01 16:20:40,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: writing mira0435601-336 from 3510272 to 3772416 tid 3 2014-07-01 16:20:40,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:695: read oid 176 snap -1 2014-07-01 16:20:40,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: expect (ObjNum 217 snap 0 seq_num 217) 2014-07-01 16:20:41,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:695: expect (ObjNum 175 snap 0 seq_num 175) 2014-07-01 16:20:41,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: finishing write tid 1 to mira0435601-89 2014-07-01 16:20:41,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: finishing write tid 2 to mira0435601-89 2014-07-01 16:20:41,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: finishing write tid 3 to mira0435601-89 2014-07-01 16:20:41,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: finishing write tid 5 to mira0435601-89 2014-07-01 16:20:41,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: finishing write tid 6 to mira0435601-89 2014-07-01 16:20:41,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 89 v 144 (ObjNum 600 snap 0 seq_num 600) dirty exists 2014-07-01 16:20:41,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: done (4 left) 2014-07-01 16:20:41,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: done (3 left) 2014-07-01 16:20:41,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:695: done (2 left) 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:696: read oid 367 snap -1 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:697: read oid 392 snap -1 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: read oid 295 snap -1 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:699: read oid 112 snap -1 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 15 current snap is 0 2014-07-01 16:20:41,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: seq_num 603 ranges {299008=360448} 2014-07-01 16:20:41,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: writing mira0435601-15 from 299008 to 659456 tid 1 2014-07-01 16:20:41,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 328 current snap is 0 2014-07-01 16:20:41,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: seq_num 604 ranges {4726784=479232,5206016=708608,5914624=94208} 2014-07-01 16:20:41,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 275 16 1346 1330 19.3417 48 0.348435 3.29719 2014-07-01 16:20:41,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: writing mira0435601-328 from 4726784 to 5206016 tid 1 2014-07-01 16:20:41,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: writing mira0435601-328 from 5206016 to 5914624 tid 2 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: writing mira0435601-328 from 5914624 to 6008832 tid 3 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 54 current snap is 0 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: seq_num 605 ranges {516096=552960,1069056=401408} 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: writing mira0435601-54 from 516096 to 1069056 tid 1 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: writing mira0435601-54 from 1069056 to 1470464 tid 2 2014-07-01 16:20:41,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: read oid 283 snap -1 2014-07-01 16:20:41,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:699: expect (ObjNum 111 snap 0 seq_num 111) 2014-07-01 16:20:42,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: expect (ObjNum 294 snap 0 seq_num 294) 2014-07-01 16:20:42,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 276 16 1361 1345 19.489 60 1.06871 3.27565 2014-07-01 16:20:42,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:696: expect (ObjNum 366 snap 0 seq_num 366) 2014-07-01 16:20:43,314.314 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 277 16 1361 1345 19.4186 0 - 3.27565 2014-07-01 16:20:44,315.315 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 278 16 1364 1348 19.3919 6 0.82559 3.27243 2014-07-01 16:20:44,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:697: expect (ObjNum 391 snap 0 seq_num 391) 2014-07-01 16:20:44,651.651 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:45,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: expect (ObjNum 282 snap 0 seq_num 282) 2014-07-01 16:20:45,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 279 16 1364 1348 19.3224 0 - 3.27243 2014-07-01 16:20:45,482.482 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:46,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: finishing write tid 1 to mira0435601-328 2014-07-01 16:20:46,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: finishing write tid 2 to mira0435601-328 2014-07-01 16:20:46,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: finishing write tid 3 to mira0435601-328 2014-07-01 16:20:46,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: finishing write tid 5 to mira0435601-328 2014-07-01 16:20:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:696: done (9 left) 2014-07-01 16:20:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:697: done (8 left) 2014-07-01 16:20:46,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: done (7 left) 2014-07-01 16:20:46,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:699: done (6 left) 2014-07-01 16:20:46,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: done (5 left) 2014-07-01 16:20:46,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 16:20:46,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: seq_num 606 ranges {622592=581632,1204224=520192,1724416=421888,2146304=663552} 2014-07-01 16:20:46,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: writing mira0435601-14 from 622592 to 1204224 tid 1 2014-07-01 16:20:46,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: writing mira0435601-14 from 1204224 to 1724416 tid 2 2014-07-01 16:20:46,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: writing mira0435601-14 from 1724416 to 2146304 tid 3 2014-07-01 16:20:46,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: writing mira0435601-14 from 2146304 to 2809856 tid 4 2014-07-01 16:20:46,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 499 current snap is 0 2014-07-01 16:20:46,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: seq_num 607 ranges {1736704=700416} 2014-07-01 16:20:46,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: writing mira0435601-499 from 1736704 to 2437120 tid 1 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: seq_num 608 ranges {2093056=724992,2818048=688128,3506176=516096} 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: writing mira0435601-313 from 2093056 to 2818048 tid 1 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: writing mira0435601-313 from 2818048 to 3506176 tid 2 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: writing mira0435601-313 from 3506176 to 4022272 tid 3 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:707: delete oid 269 current snap is 0 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: finishing write tid 6 to mira0435601-328 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 120 (ObjNum 604 snap 0 seq_num 604) dirty exists 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: finishing write tid 1 to mira0435601-15 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: finishing write tid 3 to mira0435601-15 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: finishing write tid 4 to mira0435601-15 2014-07-01 16:20:46,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 84 (ObjNum 603 snap 0 seq_num 603) dirty exists 2014-07-01 16:20:46,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:20:46.228180min lat: 0.067553 max lat: 48.8206 avg lat: 3.27716 2014-07-01 16:20:46,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:20:46,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 280 16 1371 1355 19.3534 14 4.23357 3.27716 2014-07-01 16:20:46,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: finishing write tid 1 to mira0435601-54 2014-07-01 16:20:46,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: finishing write tid 2 to mira0435601-54 2014-07-01 16:20:46,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: finishing write tid 4 to mira0435601-54 2014-07-01 16:20:46,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: finishing write tid 5 to mira0435601-54 2014-07-01 16:20:46,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 93 (ObjNum 605 snap 0 seq_num 605) dirty exists 2014-07-01 16:20:46,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: finishing write tid 1 to mira0435601-411 2014-07-01 16:20:46,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: finishing write tid 3 to mira0435601-411 2014-07-01 16:20:46,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: finishing write tid 1 to mira0435601-336 2014-07-01 16:20:46,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: finishing write tid 2 to mira0435601-336 2014-07-01 16:20:46,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: finishing write tid 3 to mira0435601-336 2014-07-01 16:20:46,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: finishing write tid 5 to mira0435601-336 2014-07-01 16:20:46,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: finishing write tid 4 to mira0435601-411 2014-07-01 16:20:46,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 95 (ObjNum 601 snap 0 seq_num 601) dirty exists 2014-07-01 16:20:46,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: finishing write tid 6 to mira0435601-336 2014-07-01 16:20:46,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 99 (ObjNum 602 snap 0 seq_num 602) dirty exists 2014-07-01 16:20:46,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 1 to mira0435601-499 2014-07-01 16:20:47,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 3 to mira0435601-499 2014-07-01 16:20:47,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: finishing write tid 1 to mira0435601-313 2014-07-01 16:20:47,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: finishing write tid 2 to mira0435601-313 2014-07-01 16:20:47,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: finishing write tid 3 to mira0435601-313 2014-07-01 16:20:47,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: finishing write tid 5 to mira0435601-313 2014-07-01 16:20:47,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 4 to mira0435601-499 2014-07-01 16:20:47,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 499 v 105 (ObjNum 607 snap 0 seq_num 607) dirty exists 2014-07-01 16:20:47,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: finishing write tid 6 to mira0435601-313 2014-07-01 16:20:47,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 120 (ObjNum 608 snap 0 seq_num 608) dirty exists 2014-07-01 16:20:47,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 281 16 1383 1367 19.4553 48 1.02685 3.28038 2014-07-01 16:20:48,385.385 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 282 16 1387 1371 19.443 16 0.976388 3.27273 2014-07-01 16:20:48,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: done (8 left) 2014-07-01 16:20:48,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: done (7 left) 2014-07-01 16:20:48,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: done (6 left) 2014-07-01 16:20:48,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: done (5 left) 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: done (4 left) 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: done (3 left) 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: done (2 left) 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:707: done (1 left) 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 1 current snap is 0 2014-07-01 16:20:48,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: seq_num 610 ranges {1015808=118784} 2014-07-01 16:20:48,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: writing mira0435601-1 from 1015808 to 1134592 tid 1 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:709: delete oid 500 current snap is 0 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 1 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 2 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 3 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 4 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 6 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: finishing write tid 7 to mira0435601-14 2014-07-01 16:20:48,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 89 (ObjNum 606 snap 0 seq_num 606) dirty exists 2014-07-01 16:20:48,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: done (2 left) 2014-07-01 16:20:48,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:709: done (1 left) 2014-07-01 16:20:48,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:710: read oid 149 snap -1 2014-07-01 16:20:48,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:711: delete oid 407 current snap is 0 2014-07-01 16:20:48,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:710: expect (ObjNum 148 snap 0 seq_num 148) 2014-07-01 16:20:49,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 283 16 1401 1385 19.5722 56 0.152419 3.25827 2014-07-01 16:20:50,319.319 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 284 16 1403 1387 19.5314 8 1.48912 3.25573 2014-07-01 16:20:50,497.497 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:50,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: finishing write tid 1 to mira0435601-1 2014-07-01 16:20:50,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: finishing write tid 3 to mira0435601-1 2014-07-01 16:20:50,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:710: done (2 left) 2014-07-01 16:20:50,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:711: done (1 left) 2014-07-01 16:20:50,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:712: delete oid 493 current snap is 0 2014-07-01 16:20:50,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: finishing write tid 4 to mira0435601-1 2014-07-01 16:20:50,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 88 (ObjNum 610 snap 0 seq_num 610) dirty exists 2014-07-01 16:20:51,059.059 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:20:35.052877', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884362072, u'health': u'HEALTH_OK', u'kb_used': 12418980, u'store_stats': {u'bytes_total': 4488119, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1342375}}, {u'last_updated': u'2014-07-01 07:19:54.527803', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887640660, u'health': u'HEALTH_OK', u'kb_used': 9140392, u'store_stats': {u'bytes_total': 11732329, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:20:01.657127', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887639380, u'health': u'HEALTH_OK', u'kb_used': 9141672, u'store_stats': {u'bytes_total': 11740564, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 37/4208 objects degraded (0.879%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 37, u'num_pgs': 144, u'data_bytes': 6364915679, u'degraded_total': 4208, u'bytes_used': 14833164288, u'op_per_sec': 5, u'write_bytes_sec': 18865530, u'version': 220, u'pgs_by_state': [{u'count': 143, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'active+recovering'}], u'read_bytes_sec': 305, u'degraded_ratio': u'0.879', u'bytes_avail': 5983465881600}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 89, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:51,060.060 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:51,315.315 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 285 16 1411 1395 19.5752 32 2.2524 3.25296 2014-07-01 16:20:51,805.805 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:20:51,805.805 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:52,316.316 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 286 16 1417 1401 19.5906 24 3.49861 3.24888 2014-07-01 16:20:52,511.511 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:53,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 287 16 1423 1407 19.6059 24 0.91963 3.24545 2014-07-01 16:20:53,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: done (1 left) 2014-07-01 16:20:53,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:712: done (0 left) 2014-07-01 16:20:53,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:713: read oid 183 snap -1 2014-07-01 16:20:53,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:714: read oid 126 snap -1 2014-07-01 16:20:53,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:715: read oid 231 snap -1 2014-07-01 16:20:53,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 330 current snap is 0 2014-07-01 16:20:53,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: seq_num 614 ranges {991232=765952} 2014-07-01 16:20:53,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: writing mira0435601-330 from 991232 to 1757184 tid 1 2014-07-01 16:20:53,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 16:20:53,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: seq_num 615 ranges {1769472=794624,2564096=557056,3121152=438272,3559424=208896} 2014-07-01 16:20:53,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: writing mira0435601-94 from 1769472 to 2564096 tid 1 2014-07-01 16:20:53,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: writing mira0435601-94 from 2564096 to 3121152 tid 2 2014-07-01 16:20:53,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: writing mira0435601-94 from 3121152 to 3559424 tid 3 2014-07-01 16:20:53,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: writing mira0435601-94 from 3559424 to 3768320 tid 4 2014-07-01 16:20:53,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:714: expect (ObjNum 125 snap 0 seq_num 125) 2014-07-01 16:20:54,316.316 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 288 16 1435 1419 19.7045 48 1.61847 3.23749 2014-07-01 16:20:54,436.436 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_large_write ... ok 2014-07-01 16:20:54,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:715: expect (ObjNum 230 snap 0 seq_num 230) 2014-07-01 16:20:55,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:713: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 16:20:55,316.316 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 289 16 1444 1428 19.7609 36 2.1969 3.22615 2014-07-01 16:20:56,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: finishing write tid 1 to mira0435601-330 2014-07-01 16:20:56,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: finishing write tid 3 to mira0435601-330 2014-07-01 16:20:56,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 1 to mira0435601-94 2014-07-01 16:20:56,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 2 to mira0435601-94 2014-07-01 16:20:56,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 3 to mira0435601-94 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 4 to mira0435601-94 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 6 to mira0435601-94 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: finishing write tid 4 to mira0435601-330 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 122 (ObjNum 614 snap 0 seq_num 614) dirty exists 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: finishing write tid 7 to mira0435601-94 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 129 (ObjNum 615 snap 0 seq_num 615) dirty exists 2014-07-01 16:20:56,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:713: done (4 left) 2014-07-01 16:20:56,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:714: done (3 left) 2014-07-01 16:20:56,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:715: done (2 left) 2014-07-01 16:20:56,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: done (1 left) 2014-07-01 16:20:56,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: done (0 left) 2014-07-01 16:20:56,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:20:56,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: seq_num 616 ranges {1081344=663552,1744896=495616,2240512=507904,2748416=200704} 2014-07-01 16:20:56,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: writing mira0435601-27 from 1081344 to 1744896 tid 1 2014-07-01 16:20:56,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: writing mira0435601-27 from 1744896 to 2240512 tid 2 2014-07-01 16:20:56,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: writing mira0435601-27 from 2240512 to 2748416 tid 3 2014-07-01 16:20:56,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: writing mira0435601-27 from 2748416 to 2949120 tid 4 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 79 current snap is 0 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: seq_num 617 ranges {204800=536576,741376=724992,1466368=479232} 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: writing mira0435601-79 from 204800 to 741376 tid 1 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: writing mira0435601-79 from 741376 to 1466368 tid 2 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: writing mira0435601-79 from 1466368 to 1945600 tid 3 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:720: read oid 139 snap -1 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 84 current snap is 0 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: seq_num 618 ranges {729088=614400,1343488=708608,2052096=94208} 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: writing mira0435601-84 from 729088 to 1343488 tid 1 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: writing mira0435601-84 from 1343488 to 2052096 tid 2 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: writing mira0435601-84 from 2052096 to 2146304 tid 3 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: read oid 206 snap -1 2014-07-01 16:20:56,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:720: expect (ObjNum 138 snap 0 seq_num 138) 2014-07-01 16:20:56,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 290 16 1457 1441 19.872 52 0.726863 3.21177 2014-07-01 16:20:56,524.524 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:56,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: expect (ObjNum 205 snap 0 seq_num 205) 2014-07-01 16:20:56,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:720: done (4 left) 2014-07-01 16:20:56,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: done (3 left) 2014-07-01 16:20:56,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 386 current snap is 0 2014-07-01 16:20:56,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: seq_num 619 ranges {1810432=483328,2293760=540672,2834432=262144} 2014-07-01 16:20:56,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: writing mira0435601-386 from 1810432 to 2293760 tid 1 2014-07-01 16:20:56,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: writing mira0435601-386 from 2293760 to 2834432 tid 2 2014-07-01 16:20:56,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: writing mira0435601-386 from 2834432 to 3096576 tid 3 2014-07-01 16:20:56,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:724: read oid 314 snap -1 2014-07-01 16:20:56,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:725: delete oid 196 current snap is 0 2014-07-01 16:20:56,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:725: done (5 left) 2014-07-01 16:20:56,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:726: read oid 457 snap -1 2014-07-01 16:20:56,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:727: read oid 132 snap -1 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 448 current snap is 0 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: seq_num 621 ranges {1974272=786432,2760704=548864,3309568=786432,4096000=270336} 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: writing mira0435601-448 from 1974272 to 2760704 tid 1 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: writing mira0435601-448 from 2760704 to 3309568 tid 2 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: writing mira0435601-448 from 3309568 to 4096000 tid 3 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: writing mira0435601-448 from 4096000 to 4366336 tid 4 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 29 current snap is 0 2014-07-01 16:20:56,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: seq_num 622 ranges {630784=655360,1286144=536576,1822720=503808,2326528=421888,2748416=143360} 2014-07-01 16:20:56,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: writing mira0435601-29 from 630784 to 1286144 tid 1 2014-07-01 16:20:56,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: writing mira0435601-29 from 1286144 to 1822720 tid 2 2014-07-01 16:20:56,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: writing mira0435601-29 from 1822720 to 2326528 tid 3 2014-07-01 16:20:57,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: writing mira0435601-29 from 2326528 to 2748416 tid 4 2014-07-01 16:20:57,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: writing mira0435601-29 from 2748416 to 2891776 tid 5 2014-07-01 16:20:57,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 16:20:57,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: seq_num 623 ranges {2330624=577536} 2014-07-01 16:20:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: writing mira0435601-391 from 2330624 to 2908160 tid 1 2014-07-01 16:20:57,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: read oid 215 snap -1 2014-07-01 16:20:57,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: read oid 40 snap -1 2014-07-01 16:20:57,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:727: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 16:20:57,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:726: expect (ObjNum 456 snap 0 seq_num 456) 2014-07-01 16:20:57,289.289 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:57,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 291 16 1470 1454 19.9824 52 0.332218 3.19325 2014-07-01 16:20:57,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:724: expect (ObjNum 546 snap 0 seq_num 546) 2014-07-01 16:20:58,161.161 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:20:58,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 292 16 1476 1460 19.9961 24 2.22105 3.18902 2014-07-01 16:20:58,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: expect (ObjNum 39 snap 0 seq_num 39) 2014-07-01 16:20:58,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: expect (ObjNum 574 snap 0 seq_num 574) 2014-07-01 16:20:58,874.874 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:20:35.052877', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884362072, u'health': u'HEALTH_OK', u'kb_used': 12418980, u'store_stats': {u'bytes_total': 4488119, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1342375}}, {u'last_updated': u'2014-07-01 07:20:54.528177', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887601444, u'health': u'HEALTH_OK', u'kb_used': 9179608, u'store_stats': {u'bytes_total': 14878057, u'bytes_log': 14614528, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 197977}}, {u'last_updated': u'2014-07-01 07:20:01.657127', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887639380, u'health': u'HEALTH_OK', u'kb_used': 9141672, u'store_stats': {u'bytes_total': 11740564, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 206212}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 26 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 144, u'data_bytes': 6645045103, u'bytes_used': 15581077504, u'op_per_sec': 12, u'version': 225, u'pgs_by_state': [{u'count': 144, u'state_name': u'active+clean'}], u'read_bytes_sec': 636294, u'write_bytes_sec': 38402516, u'bytes_avail': 5982717968384}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 89, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:20:58,875.875 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:20:58,875.875 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:20:58,875.875 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:20:58,875.875 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:20:59,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 293 16 1490 1474 20.119 56 0.334109 3.17558 2014-07-01 16:20:59,663.663 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:20:59,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: finishing write tid 1 to mira0435601-391 2014-07-01 16:20:59,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: finishing write tid 3 to mira0435601-391 2014-07-01 16:20:59,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: finishing write tid 4 to mira0435601-391 2014-07-01 16:20:59,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 115 (ObjNum 623 snap 0 seq_num 623) dirty exists 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: finishing write tid 1 to mira0435601-386 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: finishing write tid 2 to mira0435601-386 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: finishing write tid 3 to mira0435601-386 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: finishing write tid 5 to mira0435601-386 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 1 to mira0435601-448 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 2 to mira0435601-448 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 3 to mira0435601-448 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 4 to mira0435601-448 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 6 to mira0435601-448 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: finishing write tid 6 to mira0435601-386 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 148 (ObjNum 619 snap 0 seq_num 619) dirty exists 2014-07-01 16:20:59,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 1 to mira0435601-27 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 2 to mira0435601-27 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 3 to mira0435601-27 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 4 to mira0435601-27 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 6 to mira0435601-27 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: done (11 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:724: done (10 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:726: done (9 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:727: done (8 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: done (7 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: done (6 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: done (5 left) 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: read oid 442 snap -1 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: finishing write tid 1 to mira0435601-79 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: finishing write tid 7 to mira0435601-448 2014-07-01 16:20:59,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 125 (ObjNum 621 snap 0 seq_num 621) dirty exists 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: done (5 left) 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:734: read oid 485 snap -1 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: finishing write tid 2 to mira0435601-79 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: finishing write tid 3 to mira0435601-79 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: finishing write tid 1 to mira0435601-84 2014-07-01 16:20:59,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: finishing write tid 5 to mira0435601-79 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: finishing write tid 2 to mira0435601-84 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: finishing write tid 3 to mira0435601-84 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: finishing write tid 5 to mira0435601-84 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: finishing write tid 7 to mira0435601-27 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 110 (ObjNum 616 snap 0 seq_num 616) dirty exists 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: finishing write tid 6 to mira0435601-79 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 132 (ObjNum 617 snap 0 seq_num 617) dirty exists 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: finishing write tid 6 to mira0435601-84 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 126 (ObjNum 618 snap 0 seq_num 618) dirty exists 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 1 to mira0435601-29 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 2 to mira0435601-29 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 3 to mira0435601-29 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 4 to mira0435601-29 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 5 to mira0435601-29 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: done (5 left) 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: done (4 left) 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: done (3 left) 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 16:20:59,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: seq_num 624 ranges {2846720=606208,3452928=700416,4153344=503808} 2014-07-01 16:20:59,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: writing mira0435601-229 from 2846720 to 3452928 tid 1 2014-07-01 16:20:59,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: writing mira0435601-229 from 3452928 to 4153344 tid 2 2014-07-01 16:21:00,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: writing mira0435601-229 from 4153344 to 4657152 tid 3 2014-07-01 16:21:00,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 404 current snap is 0 2014-07-01 16:21:00,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: seq_num 625 ranges {385024=475136,860160=528384,1388544=675840} 2014-07-01 16:21:00,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: writing mira0435601-404 from 385024 to 860160 tid 1 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: writing mira0435601-404 from 860160 to 1388544 tid 2 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: writing mira0435601-404 from 1388544 to 2064384 tid 3 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: seq_num 626 ranges {2142208=503808} 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: writing mira0435601-400 from 2142208 to 2646016 tid 1 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 187 current snap is 0 2014-07-01 16:21:00,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: seq_num 627 ranges {659456=425984,1085440=745472,1830912=57344} 2014-07-01 16:21:00,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: writing mira0435601-187 from 659456 to 1085440 tid 1 2014-07-01 16:21:00,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: writing mira0435601-187 from 1085440 to 1830912 tid 2 2014-07-01 16:21:00,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: writing mira0435601-187 from 1830912 to 1888256 tid 3 2014-07-01 16:21:00,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: read oid 129 snap -1 2014-07-01 16:21:00,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:21:00,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: seq_num 628 ranges {704512=696320,1400832=401408} 2014-07-01 16:21:00,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: writing mira0435601-284 from 704512 to 1400832 tid 1 2014-07-01 16:21:00,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: writing mira0435601-284 from 1400832 to 1802240 tid 2 2014-07-01 16:21:00,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:21:00,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: seq_num 629 ranges {2105344=724992,2830336=512000,3342336=724992,4067328=368640} 2014-07-01 16:21:00,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: writing mira0435601-337 from 2105344 to 2830336 tid 1 2014-07-01 16:21:00,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: writing mira0435601-337 from 2830336 to 3342336 tid 2 2014-07-01 16:21:00,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: writing mira0435601-337 from 3342336 to 4067328 tid 3 2014-07-01 16:21:00,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: writing mira0435601-337 from 4067328 to 4435968 tid 4 2014-07-01 16:21:00,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:742: delete oid 10 current snap is 0 2014-07-01 16:21:00,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 7 to mira0435601-29 2014-07-01 16:21:00,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: finishing write tid 8 to mira0435601-29 2014-07-01 16:21:00,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 108 (ObjNum 622 snap 0 seq_num 622) dirty exists 2014-07-01 16:21:00,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: expect (ObjNum 441 snap 0 seq_num 441) 2014-07-01 16:21:00,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 294 16 1500 1484 20.1866 40 0.941335 3.16171 2014-07-01 16:21:00,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:734: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:21:00,519.519 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:21:00,520.520 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 46' 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: finishing write tid 1 to mira0435601-229 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: finishing write tid 2 to mira0435601-229 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: finishing write tid 3 to mira0435601-229 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: finishing write tid 5 to mira0435601-229 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: finishing write tid 6 to mira0435601-229 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 115 (ObjNum 624 snap 0 seq_num 624) dirty exists 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: finishing write tid 1 to mira0435601-404 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: finishing write tid 2 to mira0435601-404 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: finishing write tid 3 to mira0435601-404 2014-07-01 16:21:00,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: finishing write tid 5 to mira0435601-404 2014-07-01 16:21:00,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: finishing write tid 6 to mira0435601-404 2014-07-01 16:21:00,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 121 (ObjNum 625 snap 0 seq_num 625) dirty exists 2014-07-01 16:21:00,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: expect (ObjNum 128 snap 0 seq_num 128) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: done (10 left) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: done (9 left) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:734: done (8 left) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: done (7 left) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: done (6 left) 2014-07-01 16:21:01,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: done (5 left) 2014-07-01 16:21:01,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:742: done (4 left) 2014-07-01 16:21:01,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 122 current snap is 0 2014-07-01 16:21:01,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: seq_num 631 ranges {1073152=466944,1540096=724992,2265088=622592,2887680=258048} 2014-07-01 16:21:01,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: writing mira0435601-122 from 1073152 to 1540096 tid 1 2014-07-01 16:21:01,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: writing mira0435601-122 from 1540096 to 2265088 tid 2 2014-07-01 16:21:01,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: writing mira0435601-122 from 2265088 to 2887680 tid 3 2014-07-01 16:21:01,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: writing mira0435601-122 from 2887680 to 3145728 tid 4 2014-07-01 16:21:01,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 156 current snap is 0 2014-07-01 16:21:01,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: seq_num 632 ranges {3891200=544768,4435968=507904,4943872=499712,5443584=196608} 2014-07-01 16:21:01,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: writing mira0435601-156 from 3891200 to 4435968 tid 1 2014-07-01 16:21:01,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: writing mira0435601-156 from 4435968 to 4943872 tid 2 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: writing mira0435601-156 from 4943872 to 5443584 tid 3 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: writing mira0435601-156 from 5443584 to 5640192 tid 4 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 495 current snap is 0 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: seq_num 633 ranges {1085440=258048} 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: writing mira0435601-495 from 1085440 to 1343488 tid 1 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: read oid 91 snap -1 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 1 to mira0435601-156 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 340 current snap is 0 2014-07-01 16:21:01,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: seq_num 634 ranges {2252800=688128,2940928=720896,3661824=81920} 2014-07-01 16:21:01,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: writing mira0435601-340 from 2252800 to 2940928 tid 1 2014-07-01 16:21:01,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: writing mira0435601-340 from 2940928 to 3661824 tid 2 2014-07-01 16:21:01,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: writing mira0435601-340 from 3661824 to 3743744 tid 3 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:748: delete oid 264 current snap is 0 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 2 to mira0435601-156 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 3 to mira0435601-156 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 4 to mira0435601-156 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 6 to mira0435601-156 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 7 to mira0435601-156 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 98 (ObjNum 632 snap 0 seq_num 632) dirty exists 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: finishing write tid 1 to mira0435601-495 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: finishing write tid 3 to mira0435601-495 2014-07-01 16:21:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: expect (ObjNum 90 snap 0 seq_num 90) 2014-07-01 16:21:01,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 295 16 1500 1484 20.1181 0 - 3.16171 2014-07-01 16:21:02,317.317 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 296 16 1500 1484 20.0502 0 - 3.16171 2014-07-01 16:21:02,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: finishing write tid 4 to mira0435601-495 2014-07-01 16:21:02,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 131 (ObjNum 633 snap 0 seq_num 633) dirty exists 2014-07-01 16:21:03,216.216 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 46 2014-07-01 16:21:03,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: finishing write tid 1 to mira0435601-284 2014-07-01 16:21:03,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: finishing write tid 2 to mira0435601-284 2014-07-01 16:21:03,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: finishing write tid 4 to mira0435601-284 2014-07-01 16:21:03,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: finishing write tid 5 to mira0435601-284 2014-07-01 16:21:03,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 151 (ObjNum 628 snap 0 seq_num 628) dirty exists 2014-07-01 16:21:03,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: done (9 left) 2014-07-01 16:21:03,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: done (8 left) 2014-07-01 16:21:03,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: done (7 left) 2014-07-01 16:21:03,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: done (6 left) 2014-07-01 16:21:03,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:748: done (5 left) 2014-07-01 16:21:03,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: read oid 10 snap -1 2014-07-01 16:21:03,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:750: read oid 293 snap -1 2014-07-01 16:21:03,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 16:21:03,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: seq_num 636 ranges {110592=479232,589824=487424,1077248=499712,1576960=618496} 2014-07-01 16:21:03,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: writing mira0435601-382 from 110592 to 589824 tid 1 2014-07-01 16:21:03,318.318 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 297 16 1505 1489 20.05 6.66667 3.45455 3.16448 2014-07-01 16:21:03,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: writing mira0435601-382 from 589824 to 1077248 tid 2 2014-07-01 16:21:03,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: writing mira0435601-382 from 1077248 to 1576960 tid 3 2014-07-01 16:21:03,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: writing mira0435601-382 from 1576960 to 2195456 tid 4 2014-07-01 16:21:03,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 489 current snap is 0 2014-07-01 16:21:03,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: seq_num 637 ranges {1490944=507904,1998848=401408} 2014-07-01 16:21:03,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: writing mira0435601-489 from 1490944 to 1998848 tid 1 2014-07-01 16:21:03,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: writing mira0435601-489 from 1998848 to 2400256 tid 2 2014-07-01 16:21:03,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: read oid 107 snap -1 2014-07-01 16:21:03,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:750: expect (ObjNum 525 snap 0 seq_num 525) 2014-07-01 16:21:04,318.318 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 298 16 1508 1492 20.023 12 4.01802 3.16592 2014-07-01 16:21:05,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: expect (ObjNum 106 snap 0 seq_num 106) 2014-07-01 16:21:05,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 299 16 1520 1504 20.1165 48 0.127666 3.1534 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 1 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 2 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 3 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 4 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 6 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: finishing write tid 7 to mira0435601-382 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 70 (ObjNum 636 snap 0 seq_num 636) dirty exists 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: finishing write tid 1 to mira0435601-187 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: finishing write tid 2 to mira0435601-187 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: finishing write tid 1 to mira0435601-400 2014-07-01 16:21:05,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: finishing write tid 3 to mira0435601-187 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: finishing write tid 5 to mira0435601-187 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: finishing write tid 3 to mira0435601-400 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 1 to mira0435601-337 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: finishing write tid 4 to mira0435601-400 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 128 (ObjNum 626 snap 0 seq_num 626) dirty exists 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: finishing write tid 6 to mira0435601-187 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 136 (ObjNum 627 snap 0 seq_num 627) dirty exists 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 2 to mira0435601-337 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 3 to mira0435601-337 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 4 to mira0435601-337 2014-07-01 16:21:05,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 6 to mira0435601-337 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: done (9 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: done (8 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: done (7 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:750: done (6 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: done (5 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: done (4 left) 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:754: read oid 480 snap -1 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 1 to mira0435601-122 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 2 to mira0435601-122 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 3 to mira0435601-122 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 4 to mira0435601-122 2014-07-01 16:21:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 6 to mira0435601-122 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: finishing write tid 1 to mira0435601-340 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: finishing write tid 2 to mira0435601-340 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: finishing write tid 3 to mira0435601-340 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: finishing write tid 5 to mira0435601-340 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: finishing write tid 7 to mira0435601-337 2014-07-01 16:21:05,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 171 (ObjNum 629 snap 0 seq_num 629) dirty exists 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: finishing write tid 7 to mira0435601-122 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 115 (ObjNum 631 snap 0 seq_num 631) dirty exists 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 1 to mira0435601-489 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 2 to mira0435601-489 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 4 to mira0435601-489 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: finishing write tid 6 to mira0435601-340 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 112 (ObjNum 634 snap 0 seq_num 634) dirty exists 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: done (4 left) 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: done (3 left) 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: done (2 left) 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:755: read oid 173 snap -1 2014-07-01 16:21:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:756: read oid 106 snap -1 2014-07-01 16:21:05,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:757: read oid 176 snap -1 2014-07-01 16:21:05,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 22 current snap is 0 2014-07-01 16:21:05,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: seq_num 638 ranges {561152=536576,1097728=704512,1802240=442368,2244608=446464,2691072=12288} 2014-07-01 16:21:05,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: writing mira0435601-22 from 561152 to 1097728 tid 1 2014-07-01 16:21:05,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: writing mira0435601-22 from 1097728 to 1802240 tid 2 2014-07-01 16:21:05,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: writing mira0435601-22 from 1802240 to 2244608 tid 3 2014-07-01 16:21:05,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: writing mira0435601-22 from 2244608 to 2691072 tid 4 2014-07-01 16:21:05,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: writing mira0435601-22 from 2691072 to 2703360 tid 5 2014-07-01 16:21:05,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: read oid 102 snap -1 2014-07-01 16:21:05,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:757: expect (ObjNum 175 snap 0 seq_num 175) 2014-07-01 16:21:05,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:754: expect (ObjNum 479 snap 0 seq_num 479) 2014-07-01 16:21:06,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:756: expect (ObjNum 105 snap 0 seq_num 105) 2014-07-01 16:21:06,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:21:06.231729min lat: 0.067553 max lat: 48.8206 avg lat: 3.1474 2014-07-01 16:21:06,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:21:06,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 300 16 1524 1508 20.1028 16 1.01067 3.1474 2014-07-01 16:21:06,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:755: expect (ObjNum 172 snap 0 seq_num 172) 2014-07-01 16:21:06,979.979 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_list_lockers ... ok 2014-07-01 16:21:07,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 301 16 1529 1513 20.1025 20 0.072456 3.14817 2014-07-01 16:21:07,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: expect (ObjNum 519 snap 0 seq_num 519) 2014-07-01 16:21:08,236.236 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:08,236.236 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:08,237.237 INFO:teuthology.task.thrashosds.thrasher:Growing pool data 2014-07-01 16:21:08,237.237 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:21:08,318.318 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 302 16 1529 1513 20.0359 0 - 3.14817 2014-07-01 16:21:08,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 1 to mira0435601-22 2014-07-01 16:21:08,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 2 to mira0435601-22 2014-07-01 16:21:08,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 3 to mira0435601-22 2014-07-01 16:21:08,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 4 to mira0435601-22 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 5 to mira0435601-22 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 7 to mira0435601-22 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: finishing write tid 8 to mira0435601-22 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 22 v 121 (ObjNum 638 snap 0 seq_num 638) dirty exists 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 5 to mira0435601-489 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 108 (ObjNum 637 snap 0 seq_num 637) dirty exists 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: done (6 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:754: done (5 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:755: done (4 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:756: done (3 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:757: done (2 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: done (1 left) 2014-07-01 16:21:08,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: done (0 left) 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: read oid 332 snap -1 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: read oid 167 snap -1 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:762: read oid 6 snap -1 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 129 current snap is 0 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: seq_num 639 ranges {1142784=405504,1548288=532480,2080768=565248,2646016=507904} 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0435601-129 from 1142784 to 1548288 tid 1 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0435601-129 from 1548288 to 2080768 tid 2 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0435601-129 from 2080768 to 2646016 tid 3 2014-07-01 16:21:08,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0435601-129 from 2646016 to 3153920 tid 4 2014-07-01 16:21:08,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:764: read oid 131 snap -1 2014-07-01 16:21:08,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:765: read oid 78 snap -1 2014-07-01 16:21:08,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: expect (ObjNum 331 snap 0 seq_num 331) 2014-07-01 16:21:09,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:762: expect (ObjNum 528 snap 0 seq_num 528) 2014-07-01 16:21:09,030.030 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:21:09,318.318 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 303 16 1552 1536 20.2733 46 0.157437 3.14818 2014-07-01 16:21:09,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: expect (ObjNum 166 snap 0 seq_num 166) 2014-07-01 16:21:10,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:765: expect (ObjNum 77 snap 0 seq_num 77) 2014-07-01 16:21:10,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 304 16 1552 1536 20.2067 0 - 3.14818 2014-07-01 16:21:10,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:764: expect (ObjNum 130 snap 0 seq_num 130) 2014-07-01 16:21:10,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: done (5 left) 2014-07-01 16:21:10,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: done (4 left) 2014-07-01 16:21:10,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:762: done (3 left) 2014-07-01 16:21:10,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:764: done (2 left) 2014-07-01 16:21:10,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:765: done (1 left) 2014-07-01 16:21:10,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: read oid 376 snap -1 2014-07-01 16:21:10,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 192 current snap is 0 2014-07-01 16:21:10,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: seq_num 640 ranges {2240512=679936,2920448=466944,3387392=737280} 2014-07-01 16:21:10,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0435601-192 from 2240512 to 2920448 tid 1 2014-07-01 16:21:10,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0435601-192 from 2920448 to 3387392 tid 2 2014-07-01 16:21:10,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0435601-192 from 3387392 to 4124672 tid 3 2014-07-01 16:21:10,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 363 current snap is 0 2014-07-01 16:21:10,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: seq_num 641 ranges {1667072=757760,2424832=700416,3125248=102400} 2014-07-01 16:21:10,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: writing mira0435601-363 from 1667072 to 2424832 tid 1 2014-07-01 16:21:10,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: writing mira0435601-363 from 2424832 to 3125248 tid 2 2014-07-01 16:21:10,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: writing mira0435601-363 from 3125248 to 3227648 tid 3 2014-07-01 16:21:10,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 28 current snap is 0 2014-07-01 16:21:10,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: seq_num 642 ranges {954368=626688,1581056=684032,2265088=118784} 2014-07-01 16:21:10,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: writing mira0435601-28 from 954368 to 1581056 tid 1 2014-07-01 16:21:10,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: writing mira0435601-28 from 1581056 to 2265088 tid 2 2014-07-01 16:21:10,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: writing mira0435601-28 from 2265088 to 2383872 tid 3 2014-07-01 16:21:10,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: expect (ObjNum 375 snap 0 seq_num 375) 2014-07-01 16:21:11,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 305 16 1552 1536 20.1404 0 - 3.14818 2014-07-01 16:21:12,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 306 16 1573 1557 20.3491 28 0.21306 3.14009 2014-07-01 16:21:12,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: finishing write tid 1 to mira0435601-28 2014-07-01 16:21:12,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: finishing write tid 2 to mira0435601-28 2014-07-01 16:21:12,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: finishing write tid 3 to mira0435601-28 2014-07-01 16:21:12,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: finishing write tid 5 to mira0435601-28 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: finishing write tid 6 to mira0435601-28 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 28 v 102 (ObjNum 642 snap 0 seq_num 642) dirty exists 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 1 to mira0435601-192 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 2 to mira0435601-192 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 3 to mira0435601-192 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 5 to mira0435601-192 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: finishing write tid 1 to mira0435601-363 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: finishing write tid 2 to mira0435601-363 2014-07-01 16:21:12,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:770: read oid 394 snap -1 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: finishing write tid 3 to mira0435601-363 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: finishing write tid 5 to mira0435601-363 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 1 to mira0435601-129 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 6 to mira0435601-192 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 125 (ObjNum 640 snap 0 seq_num 640) dirty exists 2014-07-01 16:21:12,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: finishing write tid 6 to mira0435601-363 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 129 (ObjNum 641 snap 0 seq_num 641) dirty exists 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 2 to mira0435601-129 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 3 to mira0435601-129 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 4 to mira0435601-129 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 6 to mira0435601-129 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 7 to mira0435601-129 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 141 (ObjNum 639 snap 0 seq_num 639) dirty exists 2014-07-01 16:21:12,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: done (5 left) 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: done (4 left) 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: done (3 left) 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: done (2 left) 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: done (1 left) 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 494 current snap is 0 2014-07-01 16:21:12,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: seq_num 643 ranges {2260992=499712,2760704=757760,3518464=45056} 2014-07-01 16:21:12,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: writing mira0435601-494 from 2260992 to 2760704 tid 1 2014-07-01 16:21:12,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: writing mira0435601-494 from 2760704 to 3518464 tid 2 2014-07-01 16:21:12,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: writing mira0435601-494 from 3518464 to 3563520 tid 3 2014-07-01 16:21:12,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: read oid 275 snap -1 2014-07-01 16:21:12,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:770: expect (ObjNum 393 snap 0 seq_num 393) 2014-07-01 16:21:13,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 307 16 1573 1557 20.2828 0 - 3.14009 2014-07-01 16:21:13,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: expect (ObjNum 274 snap 0 seq_num 274) 2014-07-01 16:21:14,319.319 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 308 16 1578 1562 20.2818 10 1.85272 3.13592 2014-07-01 16:21:14,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:770: done (2 left) 2014-07-01 16:21:14,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: done (1 left) 2014-07-01 16:21:14,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:773: read oid 245 snap -1 2014-07-01 16:21:14,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: delete oid 211 current snap is 0 2014-07-01 16:21:15,154.154 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:15,154.154 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:15,154.154 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:21:15,155.155 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:21:15,155.155 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:21:15,155.155 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:21:15,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 309 16 1591 1575 20.3845 52 3.16054 3.13339 2014-07-01 16:21:16,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: finishing write tid 1 to mira0435601-494 2014-07-01 16:21:16,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: finishing write tid 2 to mira0435601-494 2014-07-01 16:21:16,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: finishing write tid 3 to mira0435601-494 2014-07-01 16:21:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: finishing write tid 5 to mira0435601-494 2014-07-01 16:21:16,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 310 16 1592 1576 20.3316 4 1.18511 3.13215 2014-07-01 16:21:16,322.322 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_list_snaps ... ok 2014-07-01 16:21:16,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: finishing write tid 6 to mira0435601-494 2014-07-01 16:21:16,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 145 (ObjNum 643 snap 0 seq_num 643) dirty exists 2014-07-01 16:21:16,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: done (2 left) 2014-07-01 16:21:16,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:773: done (1 left) 2014-07-01 16:21:16,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: done (0 left) 2014-07-01 16:21:16,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 16:21:16,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: seq_num 645 ranges {2195456=446464,2641920=401408} 2014-07-01 16:21:16,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: writing mira0435601-382 from 2195456 to 2641920 tid 1 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: writing mira0435601-382 from 2641920 to 3043328 tid 2 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:776: read oid 31 snap -1 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 66 current snap is 0 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: seq_num 646 ranges {2154496=720896} 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: writing mira0435601-66 from 2154496 to 2875392 tid 1 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 59 current snap is 0 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: seq_num 647 ranges {2396160=749568,3145728=741376,3887104=458752,4345856=4096} 2014-07-01 16:21:16,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: writing mira0435601-59 from 2396160 to 3145728 tid 1 2014-07-01 16:21:16,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: writing mira0435601-59 from 3145728 to 3887104 tid 2 2014-07-01 16:21:16,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: writing mira0435601-59 from 3887104 to 4345856 tid 3 2014-07-01 16:21:16,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: writing mira0435601-59 from 4345856 to 4349952 tid 4 2014-07-01 16:21:16,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:779: read oid 67 snap -1 2014-07-01 16:21:16,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: finishing write tid 1 to mira0435601-382 2014-07-01 16:21:16,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:776: expect (ObjNum 30 snap 0 seq_num 30) 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: finishing write tid 2 to mira0435601-382 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: finishing write tid 4 to mira0435601-382 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: finishing write tid 1 to mira0435601-66 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: finishing write tid 3 to mira0435601-66 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 1 to mira0435601-59 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 2 to mira0435601-59 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 3 to mira0435601-59 2014-07-01 16:21:16,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 4 to mira0435601-59 2014-07-01 16:21:16,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:779: expect (ObjNum 66 snap 0 seq_num 66) 2014-07-01 16:21:17,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 6 to mira0435601-59 2014-07-01 16:21:17,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:776: done (4 left) 2014-07-01 16:21:17,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:779: done (3 left) 2014-07-01 16:21:17,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: read oid 299 snap -1 2014-07-01 16:21:17,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: read oid 50 snap -1 2014-07-01 16:21:17,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: read oid 179 snap -1 2014-07-01 16:21:17,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:783: read oid 205 snap -1 2014-07-01 16:21:17,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:784: read oid 394 snap -1 2014-07-01 16:21:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 174 current snap is 0 2014-07-01 16:21:17,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: seq_num 648 ranges {196608=618496,815104=675840,1490944=528384} 2014-07-01 16:21:17,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: writing mira0435601-174 from 196608 to 815104 tid 1 2014-07-01 16:21:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: writing mira0435601-174 from 815104 to 1490944 tid 2 2014-07-01 16:21:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: writing mira0435601-174 from 1490944 to 2019328 tid 3 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 30 current snap is 0 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: seq_num 649 ranges {499712=491520,991232=503808,1495040=561152,2056192=139264} 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: writing mira0435601-30 from 499712 to 991232 tid 1 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: writing mira0435601-30 from 991232 to 1495040 tid 2 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: writing mira0435601-30 from 1495040 to 2056192 tid 3 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: writing mira0435601-30 from 2056192 to 2195456 tid 4 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: read oid 479 snap -1 2014-07-01 16:21:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:784: expect (ObjNum 393 snap 0 seq_num 393) 2014-07-01 16:21:17,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 311 16 1608 1592 20.472 64 2.1439 3.11994 2014-07-01 16:21:18,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 312 16 1615 1599 20.4961 28 1.44938 3.11279 2014-07-01 16:21:18,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: expect (ObjNum 554 snap 0 seq_num 554) 2014-07-01 16:21:19,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 313 16 1622 1606 20.52 28 1.64457 3.10569 2014-07-01 16:21:20,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 314 16 1622 1606 20.4547 0 - 3.10569 2014-07-01 16:21:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: expect (ObjNum 178 snap 0 seq_num 178) 2014-07-01 16:21:20,633.633 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:20,633.633 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:20,633.633 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:21:20,633.633 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:21:20,759.759 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_lock_unlock ... ok 2014-07-01 16:21:21,224.224 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:21:21,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 315 16 1635 1619 20.5548 26 0.362964 3.09959 2014-07-01 16:21:21,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: expect (ObjNum 49 snap 0 seq_num 49) 2014-07-01 16:21:22,278.278 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:21:22,278.278 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 56' 2014-07-01 16:21:22,320.320 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 316 16 1635 1619 20.4898 0 - 3.09959 2014-07-01 16:21:22,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 1 to mira0435601-30 2014-07-01 16:21:22,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 2 to mira0435601-30 2014-07-01 16:21:22,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:783: expect (ObjNum 204 snap 0 seq_num 204) 2014-07-01 16:21:23,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 317 16 1635 1619 20.4251 0 - 3.09959 2014-07-01 16:21:23,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 3 to mira0435601-30 2014-07-01 16:21:23,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 4 to mira0435601-30 2014-07-01 16:21:23,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 6 to mira0435601-30 2014-07-01 16:21:23,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: expect (ObjNum 478 snap 0 seq_num 478) 2014-07-01 16:21:23,739.739 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 56 2014-07-01 16:21:24,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 318 16 1642 1626 20.4489 9.33333 3.23353 3.10682 2014-07-01 16:21:25,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: finishing write tid 1 to mira0435601-174 2014-07-01 16:21:25,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: finishing write tid 2 to mira0435601-174 2014-07-01 16:21:25,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: finishing write tid 3 to mira0435601-174 2014-07-01 16:21:25,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: finishing write tid 5 to mira0435601-174 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: finishing write tid 6 to mira0435601-174 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 140 (ObjNum 648 snap 0 seq_num 648) dirty exists 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: finishing write tid 5 to mira0435601-382 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 73 (ObjNum 645 snap 0 seq_num 645) dirty exists 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: finishing write tid 4 to mira0435601-66 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 133 (ObjNum 646 snap 0 seq_num 646) dirty exists 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: finishing write tid 7 to mira0435601-59 2014-07-01 16:21:25,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 120 (ObjNum 647 snap 0 seq_num 647) dirty exists 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: done (10 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: done (9 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: done (8 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: done (7 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: done (6 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: done (5 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:783: done (4 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:784: done (3 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: done (2 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: done (1 left) 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 234 current snap is 0 2014-07-01 16:21:25,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: seq_num 650 ranges {2174976=569344,2744320=491520,3235840=311296} 2014-07-01 16:21:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: writing mira0435601-234 from 2174976 to 2744320 tid 1 2014-07-01 16:21:25,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: writing mira0435601-234 from 2744320 to 3235840 tid 2 2014-07-01 16:21:25,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: writing mira0435601-234 from 3235840 to 3547136 tid 3 2014-07-01 16:21:25,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 16:21:25,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: seq_num 651 ranges {1327104=438272,1765376=720896,2486272=81920} 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0435601-208 from 1327104 to 1765376 tid 1 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0435601-208 from 1765376 to 2486272 tid 2 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0435601-208 from 2486272 to 2568192 tid 3 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 368 current snap is 0 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: seq_num 652 ranges {2187264=65536} 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: writing mira0435601-368 from 2187264 to 2252800 tid 1 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: delete oid 300 current snap is 0 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 7 to mira0435601-30 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 107 (ObjNum 649 snap 0 seq_num 649) dirty exists 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: finishing write tid 1 to mira0435601-368 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: finishing write tid 3 to mira0435601-368 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: finishing write tid 4 to mira0435601-368 2014-07-01 16:21:25,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 109 (ObjNum 652 snap 0 seq_num 652) dirty exists 2014-07-01 16:21:25,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 319 16 1642 1626 20.3848 0 - 3.10682 2014-07-01 16:21:26,321.321 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:21:26.234866min lat: 0.067553 max lat: 48.8206 avg lat: 3.10682 2014-07-01 16:21:26,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:21:26,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 320 16 1642 1626 20.3211 0 - 3.10682 2014-07-01 16:21:27,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 321 16 1649 1633 20.345 9.33333 2.73229 3.11075 2014-07-01 16:21:27,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: finishing write tid 1 to mira0435601-234 2014-07-01 16:21:27,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: finishing write tid 2 to mira0435601-234 2014-07-01 16:21:27,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: finishing write tid 3 to mira0435601-234 2014-07-01 16:21:27,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: finishing write tid 5 to mira0435601-234 2014-07-01 16:21:27,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 1 to mira0435601-208 2014-07-01 16:21:27,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 2 to mira0435601-208 2014-07-01 16:21:27,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 3 to mira0435601-208 2014-07-01 16:21:27,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 5 to mira0435601-208 2014-07-01 16:21:27,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: done (4 left) 2014-07-01 16:21:27,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: done (3 left) 2014-07-01 16:21:27,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: done (2 left) 2014-07-01 16:21:27,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:792: delete oid 131 current snap is 0 2014-07-01 16:21:27,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:792: done (2 left) 2014-07-01 16:21:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:793: read oid 330 snap -1 2014-07-01 16:21:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 373 current snap is 0 2014-07-01 16:21:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: seq_num 655 ranges {466944=688128,1155072=544768,1699840=503808,2203648=155648} 2014-07-01 16:21:27,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: writing mira0435601-373 from 466944 to 1155072 tid 1 2014-07-01 16:21:27,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: writing mira0435601-373 from 1155072 to 1699840 tid 2 2014-07-01 16:21:27,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: writing mira0435601-373 from 1699840 to 2203648 tid 3 2014-07-01 16:21:27,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: writing mira0435601-373 from 2203648 to 2359296 tid 4 2014-07-01 16:21:27,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:795: delete oid 245 current snap is 0 2014-07-01 16:21:27,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:793: expect (ObjNum 614 snap 0 seq_num 614) 2014-07-01 16:21:28,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 322 16 1672 1656 20.5675 92 0.410035 3.10855 2014-07-01 16:21:28,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 1 to mira0435601-373 2014-07-01 16:21:28,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 2 to mira0435601-373 2014-07-01 16:21:28,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 3 to mira0435601-373 2014-07-01 16:21:28,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 4 to mira0435601-373 2014-07-01 16:21:28,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 6 to mira0435601-373 2014-07-01 16:21:28,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: finishing write tid 6 to mira0435601-234 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 175 (ObjNum 650 snap 0 seq_num 650) dirty exists 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: finishing write tid 7 to mira0435601-373 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 133 (ObjNum 655 snap 0 seq_num 655) dirty exists 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 6 to mira0435601-208 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 179 (ObjNum 651 snap 0 seq_num 651) dirty exists 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: done (4 left) 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: done (3 left) 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:793: done (2 left) 2014-07-01 16:21:28,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: done (1 left) 2014-07-01 16:21:28,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:795: done (0 left) 2014-07-01 16:21:28,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 302 current snap is 0 2014-07-01 16:21:28,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: seq_num 657 ranges {323584=430080,753664=712704,1466368=450560,1916928=40960} 2014-07-01 16:21:28,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: writing mira0435601-302 from 323584 to 753664 tid 1 2014-07-01 16:21:28,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: writing mira0435601-302 from 753664 to 1466368 tid 2 2014-07-01 16:21:28,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: writing mira0435601-302 from 1466368 to 1916928 tid 3 2014-07-01 16:21:28,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: writing mira0435601-302 from 1916928 to 1957888 tid 4 2014-07-01 16:21:28,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 88 current snap is 0 2014-07-01 16:21:28,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: seq_num 658 ranges {0=458752} 2014-07-01 16:21:28,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: writing mira0435601-88 from 0 to 458752 tid 1 2014-07-01 16:21:28,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 476 current snap is 0 2014-07-01 16:21:28,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: seq_num 659 ranges {233472=778240,1011712=401408} 2014-07-01 16:21:28,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: writing mira0435601-476 from 233472 to 1011712 tid 1 2014-07-01 16:21:28,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: writing mira0435601-476 from 1011712 to 1413120 tid 2 2014-07-01 16:21:28,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 202 current snap is 0 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: seq_num 660 ranges {2220032=651264,2871296=401408} 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: writing mira0435601-202 from 2220032 to 2871296 tid 1 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: writing mira0435601-202 from 2871296 to 3272704 tid 2 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: read oid 290 snap -1 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 1 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 2 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 3 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 4 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 6 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: finishing write tid 7 to mira0435601-302 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 125 (ObjNum 657 snap 0 seq_num 657) dirty exists 2014-07-01 16:21:28,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: finishing write tid 1 to mira0435601-88 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: finishing write tid 3 to mira0435601-88 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: finishing write tid 4 to mira0435601-88 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 135 (ObjNum 658 snap 0 seq_num 658) dirty exists 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: done (4 left) 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: done (3 left) 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:801: read oid 406 snap -1 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: read oid 240 snap -1 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:803: read oid 431 snap -1 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 127 current snap is 0 2014-07-01 16:21:28,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: seq_num 661 ranges {1597440=679936,2277376=692224,2969600=450560,3420160=462848} 2014-07-01 16:21:28,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: writing mira0435601-127 from 1597440 to 2277376 tid 1 2014-07-01 16:21:28,758.758 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:28,758.758 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:28,758.758 INFO:teuthology.task.thrashosds.thrasher:Removing osd 0, in_osds are: [3, 4, 5, 2, 0, 1] 2014-07-01 16:21:28,758.758 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 0' 2014-07-01 16:21:28,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: writing mira0435601-127 from 2277376 to 2969600 tid 2 2014-07-01 16:21:28,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: writing mira0435601-127 from 2969600 to 3420160 tid 3 2014-07-01 16:21:28,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: writing mira0435601-127 from 3420160 to 3883008 tid 4 2014-07-01 16:21:28,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:805: read oid 256 snap -1 2014-07-01 16:21:28,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: expect (ObjNum 239 snap 0 seq_num 239) 2014-07-01 16:21:29,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 16:21:29,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 323 16 1682 1666 20.6277 40 0.605766 3.09345 2014-07-01 16:21:30,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 324 16 1693 1677 20.6998 44 1.59926 3.08309 2014-07-01 16:21:30,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:801: expect (ObjNum 405 snap 0 seq_num 405) 2014-07-01 16:21:30,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:805: expect (ObjNum 255 snap 0 seq_num 255) 2014-07-01 16:21:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 1 to mira0435601-476 2014-07-01 16:21:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 2 to mira0435601-476 2014-07-01 16:21:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 4 to mira0435601-476 2014-07-01 16:21:30,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 5 to mira0435601-476 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 108 (ObjNum 659 snap 0 seq_num 659) dirty exists 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: done (7 left) 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: done (6 left) 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:801: done (5 left) 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: done (4 left) 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:803: done (3 left) 2014-07-01 16:21:30,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:805: done (2 left) 2014-07-01 16:21:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 16:21:30,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: seq_num 662 ranges {4657152=704512,5361664=401408} 2014-07-01 16:21:30,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: writing mira0435601-229 from 4657152 to 5361664 tid 1 2014-07-01 16:21:30,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: writing mira0435601-229 from 5361664 to 5763072 tid 2 2014-07-01 16:21:30,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: read oid 428 snap -1 2014-07-01 16:21:30,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: delete oid 111 current snap is 0 2014-07-01 16:21:30,963.963 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.0. 2014-07-01 16:21:30,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: finishing write tid 1 to mira0435601-229 2014-07-01 16:21:30,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: finishing write tid 2 to mira0435601-229 2014-07-01 16:21:31,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: finishing write tid 4 to mira0435601-229 2014-07-01 16:21:31,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: finishing write tid 5 to mira0435601-229 2014-07-01 16:21:31,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 128 (ObjNum 662 snap 0 seq_num 662) dirty exists 2014-07-01 16:21:31,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 325 16 1696 1680 20.673 12 0.648722 3.07862 2014-07-01 16:21:31,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: expect (ObjNum 427 snap 0 seq_num 427) 2014-07-01 16:21:32,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: finishing write tid 1 to mira0435601-202 2014-07-01 16:21:32,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: finishing write tid 2 to mira0435601-202 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: finishing write tid 4 to mira0435601-202 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 1 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 2 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 3 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 4 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 6 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: finishing write tid 5 to mira0435601-202 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 102 (ObjNum 660 snap 0 seq_num 660) dirty exists 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: finishing write tid 7 to mira0435601-127 2014-07-01 16:21:32,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 114 (ObjNum 661 snap 0 seq_num 661) dirty exists 2014-07-01 16:21:32,322.322 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 326 16 1700 1684 20.6586 16 3.12447 3.07941 2014-07-01 16:21:33,324.324 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 327 16 1711 1695 20.73 44 2.97177 3.07543 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: done (4 left) 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: done (3 left) 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: done (2 left) 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: done (1 left) 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: done (0 left) 2014-07-01 16:21:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 16:21:33,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: seq_num 664 ranges {1691648=499712,2191360=688128,2879488=516096} 2014-07-01 16:21:33,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: writing mira0435601-292 from 1691648 to 2191360 tid 1 2014-07-01 16:21:33,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: writing mira0435601-292 from 2191360 to 2879488 tid 2 2014-07-01 16:21:33,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: writing mira0435601-292 from 2879488 to 3395584 tid 3 2014-07-01 16:21:33,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 38 current snap is 0 2014-07-01 16:21:33,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: seq_num 665 ranges {442368=524288} 2014-07-01 16:21:33,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: writing mira0435601-38 from 442368 to 966656 tid 1 2014-07-01 16:21:33,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:811: delete oid 203 current snap is 0 2014-07-01 16:21:34,323.323 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 328 16 1711 1695 20.6668 0 - 3.07543 2014-07-01 16:21:35,324.324 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 329 16 1713 1697 20.6283 4 2.61458 3.07486 2014-07-01 16:21:35,974.974 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1] out_osds: [0] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:35,974.974 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:35,975.975 INFO:teuthology.task.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.501109 2014-07-01 16:21:35,976.976 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 3 0.50110913581' 2014-07-01 16:21:36,323.323 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 330 16 1715 1699 20.59 8 2.44058 3.07412 2014-07-01 16:21:37,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 331 16 1719 1703 20.5761 16 1.74952 3.07491 2014-07-01 16:21:37,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: finishing write tid 1 to mira0435601-38 2014-07-01 16:21:37,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: finishing write tid 3 to mira0435601-38 2014-07-01 16:21:38,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: finishing write tid 4 to mira0435601-38 2014-07-01 16:21:38,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 131 (ObjNum 665 snap 0 seq_num 665) dirty exists 2014-07-01 16:21:38,324.324 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 332 16 1732 1716 20.6708 52 5.35347 3.07663 2014-07-01 16:21:38,408.408 INFO:teuthology.orchestra.run.mira079.stderr:set osd.3 primary-affinity to 0.501109 (8328402) 2014-07-01 16:21:39,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 333 16 1736 1720 20.6567 16 0.562375 3.07128 2014-07-01 16:21:40,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 334 16 1736 1720 20.5949 0 - 3.07128 2014-07-01 16:21:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: finishing write tid 1 to mira0435601-292 2014-07-01 16:21:41,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: finishing write tid 2 to mira0435601-292 2014-07-01 16:21:41,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: finishing write tid 3 to mira0435601-292 2014-07-01 16:21:41,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: finishing write tid 5 to mira0435601-292 2014-07-01 16:21:41,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 335 16 1747 1731 20.6647 22 2.53185 3.07072 2014-07-01 16:21:41,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: finishing write tid 6 to mira0435601-292 2014-07-01 16:21:41,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 145 (ObjNum 664 snap 0 seq_num 664) dirty exists 2014-07-01 16:21:42,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 336 16 1749 1733 20.627 8 0.757502 3.06785 2014-07-01 16:21:43,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 337 16 1750 1734 20.5777 4 2.02189 3.06724 2014-07-01 16:21:43,418.418 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1] out_osds: [0] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:43,418.418 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:43,418.418 INFO:teuthology.task.thrashosds.thrasher:Adding osd 0 2014-07-01 16:21:43,418.418 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 0' 2014-07-01 16:21:44,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 338 16 1753 1737 20.5523 12 2.67992 3.06652 2014-07-01 16:21:44,494.494 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.0. 2014-07-01 16:21:44,506.506 INFO:teuthology.task.thrashosds.thrasher:Added osd 0 2014-07-01 16:21:45,325.325 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 339 16 1753 1737 20.4916 0 - 3.06652 2014-07-01 16:21:46,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:21:46.239179min lat: 0.067553 max lat: 48.8206 avg lat: 3.08213 2014-07-01 16:21:46,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:21:46,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 340 16 1763 1747 20.549 20 15.7537 3.08213 2014-07-01 16:21:47,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 341 16 1770 1754 20.5708 28 0.406681 3.09382 2014-07-01 16:21:48,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 342 16 1783 1767 20.6627 52 0.137544 3.08742 2014-07-01 16:21:49,328.328 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 343 16 1788 1772 20.6608 20 0.781579 3.08334 2014-07-01 16:21:49,506.506 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:49,506.506 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:49,506.506 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:21:49,506.506 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:21:49,506.506 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:21:49,507.507 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:21:50,326.326 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 344 16 1788 1772 20.6007 0 - 3.08334 2014-07-01 16:21:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: done (2 left) 2014-07-01 16:21:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: done (1 left) 2014-07-01 16:21:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:811: done (0 left) 2014-07-01 16:21:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 491 current snap is 0 2014-07-01 16:21:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: seq_num 667 ranges {434176=270336} 2014-07-01 16:21:50,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: writing mira0435601-491 from 434176 to 704512 tid 1 2014-07-01 16:21:50,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:813: read oid 72 snap -1 2014-07-01 16:21:50,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: read oid 322 snap -1 2014-07-01 16:21:50,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: read oid 370 snap -1 2014-07-01 16:21:50,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 474 current snap is 0 2014-07-01 16:21:50,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: seq_num 668 ranges {684032=745472,1429504=667648,2097152=520192,2617344=417792} 2014-07-01 16:21:50,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: writing mira0435601-474 from 684032 to 1429504 tid 1 2014-07-01 16:21:50,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: writing mira0435601-474 from 1429504 to 2097152 tid 2 2014-07-01 16:21:50,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: writing mira0435601-474 from 2097152 to 2617344 tid 3 2014-07-01 16:21:50,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: writing mira0435601-474 from 2617344 to 3035136 tid 4 2014-07-01 16:21:50,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 244 current snap is 0 2014-07-01 16:21:50,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: seq_num 669 ranges {811008=462848,1273856=401408} 2014-07-01 16:21:50,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: writing mira0435601-244 from 811008 to 1273856 tid 1 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: writing mira0435601-244 from 1273856 to 1675264 tid 2 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:818: delete oid 283 current snap is 0 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: finishing write tid 1 to mira0435601-491 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: finishing write tid 3 to mira0435601-491 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: finishing write tid 4 to mira0435601-491 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 147 (ObjNum 667 snap 0 seq_num 667) dirty exists 2014-07-01 16:21:50,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: expect (ObjNum 369 snap 0 seq_num 369) 2014-07-01 16:21:50,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 1 to mira0435601-474 2014-07-01 16:21:50,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 2 to mira0435601-474 2014-07-01 16:21:50,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 3 to mira0435601-474 2014-07-01 16:21:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 4 to mira0435601-474 2014-07-01 16:21:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 6 to mira0435601-474 2014-07-01 16:21:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: finishing write tid 7 to mira0435601-474 2014-07-01 16:21:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 108 (ObjNum 668 snap 0 seq_num 668) dirty exists 2014-07-01 16:21:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: done (6 left) 2014-07-01 16:21:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: done (5 left) 2014-07-01 16:21:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: done (4 left) 2014-07-01 16:21:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:818: done (3 left) 2014-07-01 16:21:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:819: read oid 484 snap -1 2014-07-01 16:21:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:820: delete oid 381 current snap is 0 2014-07-01 16:21:50,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:819: expect (ObjNum 483 snap 0 seq_num 483) 2014-07-01 16:21:51,327.327 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 345 16 1802 1786 20.7033 28 0.262178 3.06853 2014-07-01 16:21:52,328.328 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 346 16 1806 1790 20.6897 16 0.798941 3.06419 2014-07-01 16:21:53,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 347 16 1807 1791 20.6416 4 1.15913 3.06312 2014-07-01 16:21:53,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:813: expect (ObjNum 71 snap 0 seq_num 71) 2014-07-01 16:21:54,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: expect (ObjNum 321 snap 0 seq_num 321) 2014-07-01 16:21:54,327.327 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 348 16 1808 1792 20.5937 4 7.63289 3.06567 2014-07-01 16:21:54,986.986 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 5, 2, 1, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:21:54,986.986 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:21:54,986.986 INFO:teuthology.task.thrashosds.thrasher:Removing osd 3, in_osds are: [3, 4, 5, 2, 1, 0] 2014-07-01 16:21:54,987.987 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 3' 2014-07-01 16:21:55,327.327 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 349 16 1823 1807 20.7066 60 0.528285 3.07394 2014-07-01 16:21:55,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: finishing write tid 1 to mira0435601-244 2014-07-01 16:21:55,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: finishing write tid 2 to mira0435601-244 2014-07-01 16:21:55,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: finishing write tid 4 to mira0435601-244 2014-07-01 16:21:55,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:813: done (4 left) 2014-07-01 16:21:55,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: done (3 left) 2014-07-01 16:21:55,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:819: done (2 left) 2014-07-01 16:21:55,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:820: done (1 left) 2014-07-01 16:21:55,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: read oid 452 snap -1 2014-07-01 16:21:55,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:822: delete oid 366 current snap is 0 2014-07-01 16:21:55,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: expect (ObjNum 451 snap 0 seq_num 451) 2014-07-01 16:21:56,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 350 16 1825 1809 20.6703 8 0.572088 3.07115 2014-07-01 16:21:57,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 351 16 1825 1809 20.6114 0 - 3.07115 2014-07-01 16:21:58,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 352 16 1825 1809 20.5529 0 - 3.07115 2014-07-01 16:21:58,735.735 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.3. 2014-07-01 16:21:59,328.328 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 353 16 1835 1819 20.6079 13.3333 4.14147 3.0865 2014-07-01 16:22:00,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 354 16 1840 1824 20.6062 20 5.16383 3.08622 2014-07-01 16:22:00,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: done (2 left) 2014-07-01 16:22:00,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:822: done (1 left) 2014-07-01 16:22:00,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:22:00,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: seq_num 673 ranges {2011136=712704,2723840=479232,3203072=323584} 2014-07-01 16:22:00,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: writing mira0435601-172 from 2011136 to 2723840 tid 1 2014-07-01 16:22:00,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: writing mira0435601-172 from 2723840 to 3203072 tid 2 2014-07-01 16:22:00,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: writing mira0435601-172 from 3203072 to 3526656 tid 3 2014-07-01 16:22:00,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:824: read oid 284 snap -1 2014-07-01 16:22:00,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:825: delete oid 491 current snap is 0 2014-07-01 16:22:01,328.328 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 355 16 1847 1831 20.627 28 1.16113 3.08801 2014-07-01 16:22:01,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:825: done (3 left) 2014-07-01 16:22:01,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: read oid 225 snap -1 2014-07-01 16:22:01,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: read oid 132 snap -1 2014-07-01 16:22:01,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 16:22:01,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: seq_num 675 ranges {2342912=208896} 2014-07-01 16:22:02,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: writing mira0435601-296 from 2342912 to 2551808 tid 1 2014-07-01 16:22:02,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: read oid 48 snap -1 2014-07-01 16:22:02,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 5 current snap is 0 2014-07-01 16:22:02,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: seq_num 676 ranges {2854912=77824} 2014-07-01 16:22:02,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: writing mira0435601-5 from 2854912 to 2932736 tid 1 2014-07-01 16:22:02,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: read oid 353 snap -1 2014-07-01 16:22:02,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 264 current snap is 0 2014-07-01 16:22:02,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: seq_num 677 ranges {0=802816} 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: writing mira0435601-264 from 0 to 802816 tid 1 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: read oid 412 snap -1 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: expect (ObjNum 47 snap 0 seq_num 47) 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: done (10 left) 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:834: read oid 2 snap -1 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:835: read oid 15 snap -1 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:836: delete oid 25 current snap is 0 2014-07-01 16:22:02,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:835: expect (ObjNum 603 snap 0 seq_num 603) 2014-07-01 16:22:02,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 356 16 1857 1841 20.6814 40 3.49294 3.08413 2014-07-01 16:22:02,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: finishing write tid 1 to mira0435601-172 2014-07-01 16:22:02,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: expect (ObjNum 352 snap 0 seq_num 352) 2014-07-01 16:22:02,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: expect (ObjNum 224 snap 0 seq_num 224) 2014-07-01 16:22:03,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 357 16 1867 1851 20.7355 40 0.214698 3.07223 2014-07-01 16:22:03,751.751 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:03,751.751 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:03,751.751 INFO:teuthology.task.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2014-07-01 16:22:03,751.751 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 1 1' 2014-07-01 16:22:04,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 358 16 1871 1855 20.7223 16 0.773582 3.06916 2014-07-01 16:22:04,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 16:22:04,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:834: expect (ObjNum 1 snap 0 seq_num 1) 2014-07-01 16:22:05,306.306 INFO:teuthology.orchestra.run.mira079.stderr:set osd.1 primary-affinity to 1 (8655362) 2014-07-01 16:22:05,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 359 16 1888 1872 20.8539 68 0.484307 3.05987 2014-07-01 16:22:06,329.329 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:22:06.243163min lat: 0.067553 max lat: 48.8206 avg lat: 3.05411 2014-07-01 16:22:06,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:22:06,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 360 16 1894 1878 20.8627 24 0.18613 3.05411 2014-07-01 16:22:06,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: finishing write tid 1 to mira0435601-5 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: finishing write tid 3 to mira0435601-5 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: finishing write tid 1 to mira0435601-264 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: finishing write tid 3 to mira0435601-264 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: finishing write tid 2 to mira0435601-172 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: finishing write tid 3 to mira0435601-172 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: finishing write tid 5 to mira0435601-172 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: finishing write tid 6 to mira0435601-172 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 135 (ObjNum 673 snap 0 seq_num 673) dirty exists 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: finishing write tid 4 to mira0435601-264 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 155 (ObjNum 677 snap 0 seq_num 677) dirty exists 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: finishing write tid 4 to mira0435601-5 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 147 (ObjNum 676 snap 0 seq_num 676) dirty exists 2014-07-01 16:22:06,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:824: expect (ObjNum 628 snap 0 seq_num 628) 2014-07-01 16:22:07,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 361 16 1896 1880 20.827 8 0.637978 3.05137 2014-07-01 16:22:08,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 362 16 1897 1881 20.7805 4 1.57611 3.05059 2014-07-01 16:22:09,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 363 16 1897 1881 20.7233 0 - 3.05059 2014-07-01 16:22:10,318.318 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:10,319.319 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:10,319.319 INFO:teuthology.task.thrashosds.thrasher:Adding osd 3 2014-07-01 16:22:10,320.320 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 3' 2014-07-01 16:22:10,330.330 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 364 16 1897 1881 20.6664 0 - 3.05059 2014-07-01 16:22:11,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: expect (ObjNum 411 snap 0 seq_num 411) 2014-07-01 16:22:11,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 365 16 1912 1896 20.7741 20 6.00227 3.07435 2014-07-01 16:22:11,374.374 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.3. 2014-07-01 16:22:11,389.389 INFO:teuthology.task.thrashosds.thrasher:Added osd 3 2014-07-01 16:22:11,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: finishing write tid 5 to mira0435601-244 2014-07-01 16:22:11,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 128 (ObjNum 669 snap 0 seq_num 669) dirty exists 2014-07-01 16:22:11,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 1 to mira0435601-296 2014-07-01 16:22:11,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 3 to mira0435601-296 2014-07-01 16:22:12,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 366 16 1913 1897 20.7283 4 0.982885 3.07325 2014-07-01 16:22:13,331.331 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 367 16 1923 1907 20.7807 40 2.13761 3.07158 2014-07-01 16:22:14,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 368 16 1929 1913 20.7895 24 1.53594 3.06795 2014-07-01 16:22:15,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 369 16 1935 1919 20.7982 24 1.7059 3.06421 2014-07-01 16:22:15,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 4 to mira0435601-296 2014-07-01 16:22:15,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 130 (ObjNum 675 snap 0 seq_num 675) dirty exists 2014-07-01 16:22:16,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 370 16 1938 1922 20.7744 12 0.591904 3.06034 2014-07-01 16:22:16,389.389 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:16,390.390 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:16,390.390 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 16:22:16,390.390 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:22:16,975.975 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:22:17,348.348 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 371 16 1938 1922 20.7184 0 - 3.06034 2014-07-01 16:22:18,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: done (12 left) 2014-07-01 16:22:18,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: done (11 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:824: done (10 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: done (9 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: done (8 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: done (7 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: done (6 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: done (5 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: done (4 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: done (3 left) 2014-07-01 16:22:18,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:834: done (2 left) 2014-07-01 16:22:18,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:835: done (1 left) 2014-07-01 16:22:18,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:836: done (0 left) 2014-07-01 16:22:18,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:837: read oid 403 snap -1 2014-07-01 16:22:18,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: read oid 189 snap -1 2014-07-01 16:22:18,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:839: read oid 105 snap -1 2014-07-01 16:22:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: read oid 39 snap -1 2014-07-01 16:22:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 16:22:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: seq_num 679 ranges {2322432=700416,3022848=520192,3543040=634880,4177920=49152} 2014-07-01 16:22:18,180.180 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 24' 2014-07-01 16:22:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: writing mira0435601-58 from 2322432 to 3022848 tid 1 2014-07-01 16:22:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: writing mira0435601-58 from 3022848 to 3543040 tid 2 2014-07-01 16:22:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: writing mira0435601-58 from 3543040 to 4177920 tid 3 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: writing mira0435601-58 from 4177920 to 4227072 tid 4 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 213 current snap is 0 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: seq_num 680 ranges {0=573440,573440=454656,1028096=458752,1486848=290816} 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: writing mira0435601-213 from 0 to 573440 tid 1 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: writing mira0435601-213 from 573440 to 1028096 tid 2 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: writing mira0435601-213 from 1028096 to 1486848 tid 3 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: writing mira0435601-213 from 1486848 to 1777664 tid 4 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:843: delete oid 437 current snap is 0 2014-07-01 16:22:18,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: expect (ObjNum 38 snap 0 seq_num 38) 2014-07-01 16:22:18,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 372 16 1953 1937 20.8239 30 3.31271 3.06153 2014-07-01 16:22:18,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: expect (ObjNum 188 snap 0 seq_num 188) 2014-07-01 16:22:19,288.288 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 24 2014-07-01 16:22:19,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 373 16 1954 1938 20.7788 4 3.11369 3.06156 2014-07-01 16:22:19,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:839: expect (ObjNum 104 snap 0 seq_num 104) 2014-07-01 16:22:20,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 374 16 1954 1938 20.7233 0 - 3.06156 2014-07-01 16:22:21,332.332 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 375 16 1954 1938 20.668 0 - 3.06156 2014-07-01 16:22:22,333.333 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 376 16 1954 1938 20.6131 0 - 3.06156 2014-07-01 16:22:23,335.335 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 377 16 1954 1938 20.5584 0 - 3.06156 2014-07-01 16:22:24,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 1 to mira0435601-213 2014-07-01 16:22:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 2 to mira0435601-213 2014-07-01 16:22:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 3 to mira0435601-213 2014-07-01 16:22:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 4 to mira0435601-213 2014-07-01 16:22:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 6 to mira0435601-213 2014-07-01 16:22:24,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: finishing write tid 7 to mira0435601-213 2014-07-01 16:22:24,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 93 (ObjNum 680 snap 0 seq_num 680) dirty exists 2014-07-01 16:22:24,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 1 to mira0435601-58 2014-07-01 16:22:24,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 2 to mira0435601-58 2014-07-01 16:22:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 3 to mira0435601-58 2014-07-01 16:22:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 4 to mira0435601-58 2014-07-01 16:22:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 6 to mira0435601-58 2014-07-01 16:22:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:837: done (6 left) 2014-07-01 16:22:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: done (5 left) 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:839: done (4 left) 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: done (3 left) 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: done (2 left) 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:843: done (1 left) 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:844: read oid 329 snap -1 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 103 current snap is 0 2014-07-01 16:22:24,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: seq_num 682 ranges {1581056=520192,2101248=671744,2772992=131072} 2014-07-01 16:22:24,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: writing mira0435601-103 from 1581056 to 2101248 tid 1 2014-07-01 16:22:24,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: writing mira0435601-103 from 2101248 to 2772992 tid 2 2014-07-01 16:22:24,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: writing mira0435601-103 from 2772992 to 2904064 tid 3 2014-07-01 16:22:24,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 423 current snap is 0 2014-07-01 16:22:24,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: seq_num 683 ranges {2285568=794624,3080192=401408} 2014-07-01 16:22:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: writing mira0435601-423 from 2285568 to 3080192 tid 1 2014-07-01 16:22:24,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: writing mira0435601-423 from 3080192 to 3481600 tid 2 2014-07-01 16:22:24,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:847: read oid 434 snap -1 2014-07-01 16:22:24,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:844: expect (ObjNum 328 snap 0 seq_num 328) 2014-07-01 16:22:24,298.298 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:24,298.298 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:24,299.299 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 16:22:24,299.299 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:22:24,334.334 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 378 16 1972 1956 20.6944 14.4 0.192928 3.08391 2014-07-01 16:22:24,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: finishing write tid 7 to mira0435601-58 2014-07-01 16:22:24,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 70 (ObjNum 679 snap 0 seq_num 679) dirty exists 2014-07-01 16:22:24,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: finishing write tid 1 to mira0435601-103 2014-07-01 16:22:24,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:847: expect (ObjNum 580 snap 0 seq_num 580) 2014-07-01 16:22:25,080.080 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:22:25,333.333 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 379 16 1976 1960 20.682 16 0.10714 3.07838 2014-07-01 16:22:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: done (4 left) 2014-07-01 16:22:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:844: done (3 left) 2014-07-01 16:22:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:847: done (2 left) 2014-07-01 16:22:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 146 current snap is 0 2014-07-01 16:22:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: seq_num 684 ranges {892928=20480} 2014-07-01 16:22:25,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: writing mira0435601-146 from 892928 to 913408 tid 1 2014-07-01 16:22:25,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: read oid 200 snap -1 2014-07-01 16:22:25,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:850: read oid 334 snap -1 2014-07-01 16:22:25,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: delete oid 166 current snap is 0 2014-07-01 16:22:25,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: expect (ObjNum 199 snap 0 seq_num 199) 2014-07-01 16:22:25,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:850: expect (ObjNum 532 snap 0 seq_num 532) 2014-07-01 16:22:26,120.120 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 26' 2014-07-01 16:22:26,333.333 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:22:26.247051min lat: 0.067553 max lat: 48.8206 avg lat: 3.07838 2014-07-01 16:22:26,333.333 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:22:26,334.334 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 380 16 1976 1960 20.6276 0 - 3.07838 2014-07-01 16:22:27,333.333 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 381 16 1976 1960 20.5735 0 - 3.07838 2014-07-01 16:22:28,334.334 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 382 16 1976 1960 20.5196 0 - 3.07838 2014-07-01 16:22:29,335.335 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 383 16 1976 1960 20.466 0 - 3.07838 2014-07-01 16:22:29,683.683 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 26 2014-07-01 16:22:29,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: finishing write tid 1 to mira0435601-423 2014-07-01 16:22:29,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: finishing write tid 2 to mira0435601-423 2014-07-01 16:22:29,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: finishing write tid 4 to mira0435601-423 2014-07-01 16:22:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: finishing write tid 5 to mira0435601-423 2014-07-01 16:22:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 138 (ObjNum 683 snap 0 seq_num 683) dirty exists 2014-07-01 16:22:30,334.334 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 384 16 1988 1972 20.5377 9.6 0.088596 3.09219 2014-07-01 16:22:31,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: finishing write tid 1 to mira0435601-146 2014-07-01 16:22:31,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: finishing write tid 3 to mira0435601-146 2014-07-01 16:22:31,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: finishing write tid 4 to mira0435601-146 2014-07-01 16:22:31,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 110 (ObjNum 684 snap 0 seq_num 684) dirty exists 2014-07-01 16:22:31,336.336 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 385 16 2004 1988 20.6506 64 0.26344 3.09596 2014-07-01 16:22:32,334.334 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 386 16 2005 1989 20.6074 4 0.602864 3.0947 2014-07-01 16:22:33,337.337 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 387 16 2016 2000 20.6679 44 0.403981 3.08785 2014-07-01 16:22:34,335.335 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 388 16 2027 2011 20.728 44 0.624384 3.07849 2014-07-01 16:22:34,696.696 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:34,696.696 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:34,696.696 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 16:22:34,696.696 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:22:34,696.696 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:22:34,696.696 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 16:22:35,336.336 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 389 16 2048 2032 20.8906 84 1.8727 3.05864 2014-07-01 16:22:35,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: done (5 left) 2014-07-01 16:22:35,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: done (4 left) 2014-07-01 16:22:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: done (3 left) 2014-07-01 16:22:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:850: done (2 left) 2014-07-01 16:22:35,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: done (1 left) 2014-07-01 16:22:35,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:852: read oid 437 snap -1 2014-07-01 16:22:35,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: read oid 409 snap -1 2014-07-01 16:22:35,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:854: read oid 255 snap -1 2014-07-01 16:22:35,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 16:22:35,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: seq_num 686 ranges {2490368=614400} 2014-07-01 16:22:35,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: writing mira0435601-223 from 2490368 to 3104768 tid 1 2014-07-01 16:22:35,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 16:22:35,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: seq_num 687 ranges {3571712=643072,4214784=724992,4939776=479232} 2014-07-01 16:22:35,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: writing mira0435601-250 from 3571712 to 4214784 tid 1 2014-07-01 16:22:35,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: writing mira0435601-250 from 4214784 to 4939776 tid 2 2014-07-01 16:22:35,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: writing mira0435601-250 from 4939776 to 5419008 tid 3 2014-07-01 16:22:35,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: delete oid 57 current snap is 0 2014-07-01 16:22:35,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: expect (ObjNum 408 snap 0 seq_num 408) 2014-07-01 16:22:36,335.335 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 390 16 2049 2033 20.8473 4 0.744649 3.0575 2014-07-01 16:22:37,335.335 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 391 16 2053 2037 20.8349 16 2.48527 3.05616 2014-07-01 16:22:38,337.337 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 392 16 2054 2038 20.7919 4 2.10125 3.05569 2014-07-01 16:22:39,337.337 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 393 16 2054 2038 20.739 0 - 3.05569 2014-07-01 16:22:40,170.170 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:40,171.171 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:40,171.171 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 16:22:40,172.172 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:22:40,336.336 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 394 16 2059 2043 20.7371 10 5.1506 3.06042 2014-07-01 16:22:40,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: finishing write tid 1 to mira0435601-223 2014-07-01 16:22:40,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: finishing write tid 3 to mira0435601-223 2014-07-01 16:22:40,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: finishing write tid 4 to mira0435601-223 2014-07-01 16:22:40,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 114 (ObjNum 686 snap 0 seq_num 686) dirty exists 2014-07-01 16:22:40,860.860 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:22:41,337.337 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 395 16 2067 2051 20.7656 32 1.29513 3.06504 2014-07-01 16:22:41,892.892 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:22:41,892.892 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pg_num 44' 2014-07-01 16:22:42,338.338 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 396 16 2082 2066 20.8647 60 0.351763 3.05955 2014-07-01 16:22:43,001.001 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pg_num to 44 2014-07-01 16:22:43,338.338 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 397 16 2096 2080 20.9531 56 0.372604 3.05011 2014-07-01 16:22:44,338.338 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 398 16 2111 2095 21.0512 60 0.449197 3.03602 2014-07-01 16:22:45,339.339 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 399 16 2122 2106 21.1087 44 0.629826 3.02518 2014-07-01 16:22:46,339.339 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:22:46.251299min lat: 0.067553 max lat: 48.8206 avg lat: 3.02447 2014-07-01 16:22:46,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:22:46,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 400 16 2123 2107 21.0659 4 1.52725 3.02447 2014-07-01 16:22:47,338.338 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 401 16 2124 2108 21.0234 4 1.63428 3.02381 2014-07-01 16:22:48,009.009 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:48,009.009 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:48,009.009 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2014-07-01 16:22:48,009.009 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 1' 2014-07-01 16:22:48,338.338 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 402 16 2131 2115 21.0407 28 0.057269 3.02042 2014-07-01 16:22:49,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 403 16 2137 2121 21.048 24 0.217436 3.01721 2014-07-01 16:22:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: finishing write tid 1 to mira0435601-250 2014-07-01 16:22:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: finishing write tid 2 to mira0435601-250 2014-07-01 16:22:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: finishing write tid 3 to mira0435601-250 2014-07-01 16:22:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: finishing write tid 5 to mira0435601-250 2014-07-01 16:22:50,002.002 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 1 (8655362) 2014-07-01 16:22:50,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: finishing write tid 6 to mira0435601-250 2014-07-01 16:22:50,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 90 (ObjNum 687 snap 0 seq_num 687) dirty exists 2014-07-01 16:22:50,339.339 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 404 16 2150 2134 21.1246 52 5.4659 3.01881 2014-07-01 16:22:51,339.339 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 405 16 2150 2134 21.0725 0 - 3.01881 2014-07-01 16:22:52,341.341 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 406 16 2150 2134 21.0206 0 - 3.01881 2014-07-01 16:22:53,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 407 16 2151 2135 20.9787 1.33333 2.95536 3.01878 2014-07-01 16:22:53,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:854: expect (ObjNum 254 snap 0 seq_num 254) 2014-07-01 16:22:54,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 408 16 2167 2151 21.0842 64 7.86912 3.02065 2014-07-01 16:22:54,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: finishing write tid 2 to mira0435601-103 2014-07-01 16:22:54,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: finishing write tid 3 to mira0435601-103 2014-07-01 16:22:54,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: finishing write tid 5 to mira0435601-103 2014-07-01 16:22:55,009.009 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:55,010.010 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:22:55,010.010 INFO:teuthology.task.thrashosds.thrasher:Killing osd 4, live_osds are [1, 0, 3, 2, 4, 5] 2014-07-01 16:22:55,010.010 DEBUG:teuthology.task.ceph.osd.4:waiting for process to exit 2014-07-01 16:22:55,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 409 16 2177 2161 21.1304 40 0.559692 3.01966 2014-07-01 16:22:55,672.672 INFO:teuthology.task.radosbench.radosbench.0.mira043.stderr:2014-07-01 07:22:55.585780 7f8daadf8700 0 -- 10.214.133.128:0/1005625 >> 10.214.132.120:6805/6415 pipe(0x7f8dac022860 sd=14 :0 s=1 pgs=0 cs=0 l=1 c=0x7f8dac022ad0).fault 2014-07-01 16:22:55,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: finishing write tid 6 to mira0435601-103 2014-07-01 16:22:55,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 114 (ObjNum 682 snap 0 seq_num 682) dirty exists 2014-07-01 16:22:56,341.341 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 410 16 2191 2175 21.2154 56 0.866868 3.01121 2014-07-01 16:22:57,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 411 16 2192 2176 21.1735 4 0.90523 3.01025 2014-07-01 16:22:57,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: done (6 left) 2014-07-01 16:22:57,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:852: done (5 left) 2014-07-01 16:22:57,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: done (4 left) 2014-07-01 16:22:57,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:854: done (3 left) 2014-07-01 16:22:57,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: done (2 left) 2014-07-01 16:22:57,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: done (1 left) 2014-07-01 16:22:57,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: done (0 left) 2014-07-01 16:22:57,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:858: read oid 178 snap -1 2014-07-01 16:22:57,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 61 current snap is 0 2014-07-01 16:22:57,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: seq_num 689 ranges {1294336=786432,2080768=491520,2572288=311296} 2014-07-01 16:22:57,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: writing mira0435601-61 from 1294336 to 2080768 tid 1 2014-07-01 16:22:57,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: writing mira0435601-61 from 2080768 to 2572288 tid 2 2014-07-01 16:22:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: writing mira0435601-61 from 2572288 to 2883584 tid 3 2014-07-01 16:22:57,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 430 current snap is 0 2014-07-01 16:22:57,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: seq_num 690 ranges {2355200=462848,2818048=475136,3293184=327680} 2014-07-01 16:22:57,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: writing mira0435601-430 from 2355200 to 2818048 tid 1 2014-07-01 16:22:57,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: writing mira0435601-430 from 2818048 to 3293184 tid 2 2014-07-01 16:22:57,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: writing mira0435601-430 from 3293184 to 3620864 tid 3 2014-07-01 16:22:57,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 16:22:57,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: seq_num 691 ranges {3104768=86016} 2014-07-01 16:22:57,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: writing mira0435601-223 from 3104768 to 3190784 tid 1 2014-07-01 16:22:57,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: read oid 118 snap -1 2014-07-01 16:22:57,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:858: expect (ObjNum 177 snap 0 seq_num 177) 2014-07-01 16:22:58,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:858: done (4 left) 2014-07-01 16:22:58,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 45 current snap is 0 2014-07-01 16:22:58,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: seq_num 692 ranges {704512=606208,1310720=401408} 2014-07-01 16:22:58,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 412 16 2198 2182 21.1804 24 1.82982 3.00733 2014-07-01 16:22:58,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: writing mira0435601-45 from 704512 to 1310720 tid 1 2014-07-01 16:22:58,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: writing mira0435601-45 from 1310720 to 1712128 tid 2 2014-07-01 16:22:58,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 225 current snap is 0 2014-07-01 16:22:58,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: seq_num 693 ranges {2363392=634880,2998272=471040,3469312=471040,3940352=536576,4476928=126976} 2014-07-01 16:22:58,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0435601-225 from 2363392 to 2998272 tid 1 2014-07-01 16:22:58,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0435601-225 from 2998272 to 3469312 tid 2 2014-07-01 16:22:58,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0435601-225 from 3469312 to 3940352 tid 3 2014-07-01 16:22:58,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0435601-225 from 3940352 to 4476928 tid 4 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0435601-225 from 4476928 to 4603904 tid 5 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:865: read oid 100 snap -1 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:866: read oid 301 snap -1 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: read oid 172 snap -1 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 343 current snap is 0 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: seq_num 694 ranges {1449984=659456,2109440=401408} 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: writing mira0435601-343 from 1449984 to 2109440 tid 1 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: writing mira0435601-343 from 2109440 to 2510848 tid 2 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 475 current snap is 0 2014-07-01 16:22:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: seq_num 695 ranges {360448=425984} 2014-07-01 16:22:58,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: writing mira0435601-475 from 360448 to 786432 tid 1 2014-07-01 16:22:58,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:870: read oid 356 snap -1 2014-07-01 16:22:58,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 310 current snap is 0 2014-07-01 16:22:58,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: seq_num 696 ranges {266240=450560,716800=671744,1388544=532480} 2014-07-01 16:22:58,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: writing mira0435601-310 from 266240 to 716800 tid 1 2014-07-01 16:22:58,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: writing mira0435601-310 from 716800 to 1388544 tid 2 2014-07-01 16:22:58,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: writing mira0435601-310 from 1388544 to 1921024 tid 3 2014-07-01 16:22:58,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:872: read oid 33 snap -1 2014-07-01 16:22:58,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:865: expect (ObjNum 99 snap 0 seq_num 99) 2014-07-01 16:22:59,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 413 16 2198 2182 21.1291 0 - 3.00733 2014-07-01 16:23:00,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:865: done (13 left) 2014-07-01 16:23:00,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:872: done (12 left) 2014-07-01 16:23:00,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:873: read oid 15 snap -1 2014-07-01 16:23:00,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: read oid 246 snap -1 2014-07-01 16:23:00,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:875: read oid 58 snap -1 2014-07-01 16:23:00,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 16:23:00,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: seq_num 697 ranges {2199552=479232} 2014-07-01 16:23:00,340.340 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 414 16 2198 2182 21.078 0 - 3.00733 2014-07-01 16:23:00,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: writing mira0435601-273 from 2199552 to 2678784 tid 1 2014-07-01 16:23:00,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:01,010.010 INFO:teuthology.task.ceph.osd.4:Stopped 2014-07-01 16:23:01,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 415 16 2202 2186 21.0658 5.33333 5.83025 3.01345 2014-07-01 16:23:02,341.341 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 416 16 2206 2190 21.0536 16 0.203799 3.01253 2014-07-01 16:23:03,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: finishing write tid 1 to mira0435601-430 2014-07-01 16:23:03,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: finishing write tid 2 to mira0435601-430 2014-07-01 16:23:03,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: finishing write tid 3 to mira0435601-430 2014-07-01 16:23:03,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: finishing write tid 5 to mira0435601-430 2014-07-01 16:23:03,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: finishing write tid 1 to mira0435601-475 2014-07-01 16:23:03,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: finishing write tid 3 to mira0435601-475 2014-07-01 16:23:03,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 1 to mira0435601-45 2014-07-01 16:23:03,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 2 to mira0435601-45 2014-07-01 16:23:03,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 4 to mira0435601-45 2014-07-01 16:23:03,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: expect (ObjNum 117 snap 0 seq_num 117) 2014-07-01 16:23:03,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 417 16 2210 2194 21.0415 16 2.00519 3.01202 2014-07-01 16:23:04,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: finishing write tid 4 to mira0435601-475 2014-07-01 16:23:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 150 (ObjNum 695 snap 0 seq_num 695) dirty exists 2014-07-01 16:23:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 5 to mira0435601-45 2014-07-01 16:23:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 115 (ObjNum 692 snap 0 seq_num 692) dirty exists 2014-07-01 16:23:04,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:866: expect (ObjNum 300 snap 0 seq_num 300) 2014-07-01 16:23:04,341.341 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 418 16 2218 2202 21.0677 32 0.376572 3.00337 2014-07-01 16:23:05,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: expect (ObjNum 673 snap 0 seq_num 673) 2014-07-01 16:23:05,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 419 16 2223 2207 21.0651 20 1.1737 2.99869 2014-07-01 16:23:06,011.011 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 3] out_osds: [] dead_osds: [4] live_osds: [1, 0, 3, 2, 5] 2014-07-01 16:23:06,011.011 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 4 2014-07-01 16:23:06,011.011 INFO:teuthology.task.ceph.osd.4:Restarting daemon 2014-07-01 16:23:06,011.011 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 4' 2014-07-01 16:23:06,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:23:06.255153min lat: 0.057269 max lat: 48.8206 avg lat: 2.98887 2014-07-01 16:23:06,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:23:06,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 420 16 2231 2215 21.0912 32 0.401358 2.98887 2014-07-01 16:23:06,381.381 INFO:teuthology.task.ceph.osd.4:Started 2014-07-01 16:23:06,381.381 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok dump_ops_in_flight' 2014-07-01 16:23:06,407.407 INFO:teuthology.task.ceph.osd.4.mira028.stdout:starting osd.4 at :/0 osd_data /var/lib/ceph/osd/ceph-4 /var/lib/ceph/osd/ceph-4/journal 2014-07-01 16:23:06,585.585 INFO:teuthology.task.ceph.osd.4.mira028.stderr:2014-07-01 07:23:06.439165 7fe1fe6cc780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:no valid command found; 10 closest matches: 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:config show 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:help 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:log dump 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:get_command_descriptions 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:git_version 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:config set [...] 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:version 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:2 2014-07-01 16:23:06,841.841 INFO:teuthology.orchestra.run.mira028.stderr:config get 2014-07-01 16:23:06,842.842 INFO:teuthology.orchestra.run.mira028.stderr:0 2014-07-01 16:23:06,842.842 INFO:teuthology.orchestra.run.mira028.stderr:admin_socket: invalid command 2014-07-01 16:23:06,848.848 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 4, ['dump_ops_in_flight'] 2014-07-01 16:23:07,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:870: expect (ObjNum 355 snap 0 seq_num 355) 2014-07-01 16:23:07,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 421 16 2231 2215 21.0411 0 - 2.98887 2014-07-01 16:23:07,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: finishing write tid 6 to mira0435601-430 2014-07-01 16:23:07,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 134 (ObjNum 690 snap 0 seq_num 690) dirty exists 2014-07-01 16:23:07,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:875: expect (ObjNum 679 snap 0 seq_num 679) 2014-07-01 16:23:08,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 422 16 2231 2215 20.9912 0 - 2.98887 2014-07-01 16:23:09,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 423 16 2231 2215 20.9416 0 - 2.98887 2014-07-01 16:23:09,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: finishing write tid 1 to mira0435601-343 2014-07-01 16:23:09,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:873: expect (ObjNum 603 snap 0 seq_num 603) 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: finishing write tid 2 to mira0435601-343 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: finishing write tid 4 to mira0435601-343 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: finishing write tid 5 to mira0435601-343 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 138 (ObjNum 694 snap 0 seq_num 694) dirty exists 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: done (15 left) 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: done (14 left) 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: done (13 left) 2014-07-01 16:23:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:866: done (12 left) 2014-07-01 16:23:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: done (11 left) 2014-07-01 16:23:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: done (10 left) 2014-07-01 16:23:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: done (9 left) 2014-07-01 16:23:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:870: done (8 left) 2014-07-01 16:23:10,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:873: done (7 left) 2014-07-01 16:23:10,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:875: done (6 left) 2014-07-01 16:23:10,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:877: read oid 281 snap -1 2014-07-01 16:23:10,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 373 current snap is 0 2014-07-01 16:23:10,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: seq_num 698 ranges {2359296=598016,2957312=684032,3641344=118784} 2014-07-01 16:23:10,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0435601-373 from 2359296 to 2957312 tid 1 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0435601-373 from 2957312 to 3641344 tid 2 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0435601-373 from 3641344 to 3760128 tid 3 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:879: delete oid 166 current snap is 0 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:879: done (8 left) 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:880: read oid 432 snap -1 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: seq_num 700 ranges {1953792=700416,2654208=499712,3153920=536576,3690496=495616,4186112=73728} 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: writing mira0435601-409 from 1953792 to 2654208 tid 1 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: writing mira0435601-409 from 2654208 to 3153920 tid 2 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: writing mira0435601-409 from 3153920 to 3690496 tid 3 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: writing mira0435601-409 from 3690496 to 4186112 tid 4 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: writing mira0435601-409 from 4186112 to 4259840 tid 5 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 419 current snap is 0 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: seq_num 701 ranges {589824=573440,1163264=679936,1843200=765952,2609152=159744} 2014-07-01 16:23:10,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: writing mira0435601-419 from 589824 to 1163264 tid 1 2014-07-01 16:23:10,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: writing mira0435601-419 from 1163264 to 1843200 tid 2 2014-07-01 16:23:10,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: writing mira0435601-419 from 1843200 to 2609152 tid 3 2014-07-01 16:23:10,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: writing mira0435601-419 from 2609152 to 2768896 tid 4 2014-07-01 16:23:10,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 16:23:10,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: seq_num 702 ranges {1515520=442368,1957888=663552,2621440=483328,3104768=458752} 2014-07-01 16:23:10,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 424 16 2231 2215 20.8922 0 - 2.98887 2014-07-01 16:23:10,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: writing mira0435601-435 from 1515520 to 1957888 tid 1 2014-07-01 16:23:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: writing mira0435601-435 from 1957888 to 2621440 tid 2 2014-07-01 16:23:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: writing mira0435601-435 from 2621440 to 3104768 tid 3 2014-07-01 16:23:10,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: writing mira0435601-435 from 3104768 to 3563520 tid 4 2014-07-01 16:23:10,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 55 current snap is 0 2014-07-01 16:23:10,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: seq_num 703 ranges {1748992=471040,2220032=401408} 2014-07-01 16:23:10,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: writing mira0435601-55 from 1748992 to 2220032 tid 1 2014-07-01 16:23:10,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: writing mira0435601-55 from 2220032 to 2621440 tid 2 2014-07-01 16:23:10,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: read oid 110 snap -1 2014-07-01 16:23:10,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 135 current snap is 0 2014-07-01 16:23:10,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: seq_num 704 ranges {1536000=794624,2330624=479232,2809856=323584} 2014-07-01 16:23:10,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: writing mira0435601-135 from 1536000 to 2330624 tid 1 2014-07-01 16:23:10,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: writing mira0435601-135 from 2330624 to 2809856 tid 2 2014-07-01 16:23:10,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: writing mira0435601-135 from 2809856 to 3133440 tid 3 2014-07-01 16:23:10,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 16:23:10,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: seq_num 705 ranges {0=663552,663552=659456,1323008=143360} 2014-07-01 16:23:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: writing mira0435601-124 from 0 to 663552 tid 1 2014-07-01 16:23:10,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: writing mira0435601-124 from 663552 to 1323008 tid 2 2014-07-01 16:23:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: writing mira0435601-124 from 1323008 to 1466368 tid 3 2014-07-01 16:23:10,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:10,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: finishing write tid 1 to mira0435601-55 2014-07-01 16:23:10,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: finishing write tid 2 to mira0435601-55 2014-07-01 16:23:10,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: finishing write tid 4 to mira0435601-55 2014-07-01 16:23:10,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: expect (ObjNum 109 snap 0 seq_num 109) 2014-07-01 16:23:10,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 1 to mira0435601-61 2014-07-01 16:23:10,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 1 to mira0435601-419 2014-07-01 16:23:10,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: expect (ObjNum 245 snap 0 seq_num 245) 2014-07-01 16:23:11,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 425 16 2239 2223 20.9183 6.4 0.145192 3.0044 2014-07-01 16:23:11,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 2 to mira0435601-419 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: finishing write tid 5 to mira0435601-55 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 93 (ObjNum 703 snap 0 seq_num 703) dirty exists 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: done (15 left) 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: done (14 left) 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: done (13 left) 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: read oid 32 snap -1 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 2 to mira0435601-61 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 3 to mira0435601-419 2014-07-01 16:23:11,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:880: expect (ObjNum 431 snap 0 seq_num 431) 2014-07-01 16:23:11,849.849 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok dump_ops_in_flight' 2014-07-01 16:23:12,338.338 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:23:12,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 426 16 2265 2249 21.1133 104 0.460612 3.02893 2014-07-01 16:23:12,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 3 to mira0435601-61 2014-07-01 16:23:12,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 4 to mira0435601-419 2014-07-01 16:23:12,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:880: done (13 left) 2014-07-01 16:23:12,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 348 current snap is 0 2014-07-01 16:23:12,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: seq_num 706 ranges {610304=290816} 2014-07-01 16:23:13,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: writing mira0435601-348 from 610304 to 901120 tid 1 2014-07-01 16:23:13,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 341 current snap is 0 2014-07-01 16:23:13,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: seq_num 707 ranges {1396736=163840} 2014-07-01 16:23:13,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: writing mira0435601-341 from 1396736 to 1560576 tid 1 2014-07-01 16:23:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 191 current snap is 0 2014-07-01 16:23:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: seq_num 708 ranges {0=483328,483328=401408} 2014-07-01 16:23:13,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: writing mira0435601-191 from 0 to 483328 tid 1 2014-07-01 16:23:13,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: writing mira0435601-191 from 483328 to 884736 tid 2 2014-07-01 16:23:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 5 to mira0435601-61 2014-07-01 16:23:13,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 6 to mira0435601-419 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 7 to mira0435601-419 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 153 (ObjNum 701 snap 0 seq_num 701) dirty exists 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 1 to mira0435601-435 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: done (15 left) 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 498 current snap is 0 2014-07-01 16:23:13,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: seq_num 709 ranges {503808=512000,1015808=483328,1499136=708608,2207744=413696} 2014-07-01 16:23:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: writing mira0435601-498 from 503808 to 1015808 tid 1 2014-07-01 16:23:13,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: writing mira0435601-498 from 1015808 to 1499136 tid 2 2014-07-01 16:23:13,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: writing mira0435601-498 from 1499136 to 2207744 tid 3 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: writing mira0435601-498 from 2207744 to 2621440 tid 4 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: finishing write tid 1 to mira0435601-135 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 2 to mira0435601-435 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 1 to mira0435601-223 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: finishing write tid 1 to mira0435601-310 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 3 to mira0435601-223 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 3 to mira0435601-435 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: finishing write tid 2 to mira0435601-135 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 4 to mira0435601-435 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: finishing write tid 3 to mira0435601-135 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: finishing write tid 1 to mira0435601-273 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: finishing write tid 5 to mira0435601-135 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: finishing write tid 3 to mira0435601-273 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: finishing write tid 6 to mira0435601-135 2014-07-01 16:23:13,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 135 v 79 (ObjNum 704 snap 0 seq_num 704) dirty exists 2014-07-01 16:23:13,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 4 to mira0435601-223 2014-07-01 16:23:13,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 118 (ObjNum 691 snap 0 seq_num 691) dirty exists 2014-07-01 16:23:13,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 6 to mira0435601-435 2014-07-01 16:23:13,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: done (15 left) 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: done (14 left) 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: delete oid 388 current snap is 0 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 1 to mira0435601-225 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 2 to mira0435601-225 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 3 to mira0435601-225 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: finishing write tid 4 to mira0435601-273 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 114 (ObjNum 697 snap 0 seq_num 697) dirty exists 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 1 to mira0435601-409 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 2 to mira0435601-409 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 3 to mira0435601-409 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 4 to mira0435601-225 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 6 to mira0435601-61 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 188 (ObjNum 689 snap 0 seq_num 689) dirty exists 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 5 to mira0435601-225 2014-07-01 16:23:13,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 1 to mira0435601-373 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 2 to mira0435601-373 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 3 to mira0435601-373 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 5 to mira0435601-373 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 6 to mira0435601-373 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 137 (ObjNum 698 snap 0 seq_num 698) dirty exists 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 4 to mira0435601-409 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 5 to mira0435601-409 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 7 to mira0435601-409 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: finishing write tid 8 to mira0435601-409 2014-07-01 16:23:13,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 163 (ObjNum 700 snap 0 seq_num 700) dirty exists 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 7 to mira0435601-225 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: finishing write tid 7 to mira0435601-435 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 193 (ObjNum 702 snap 0 seq_num 702) dirty exists 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: finishing write tid 2 to mira0435601-310 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: finishing write tid 3 to mira0435601-310 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: finishing write tid 5 to mira0435601-310 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: finishing write tid 1 to mira0435601-124 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: finishing write tid 2 to mira0435601-124 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: finishing write tid 3 to mira0435601-124 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: finishing write tid 5 to mira0435601-124 2014-07-01 16:23:13,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 8 to mira0435601-225 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 128 (ObjNum 693 snap 0 seq_num 693) dirty exists 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: finishing write tid 6 to mira0435601-310 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 122 (ObjNum 696 snap 0 seq_num 696) dirty exists 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: finishing write tid 6 to mira0435601-124 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 126 (ObjNum 705 snap 0 seq_num 705) dirty exists 2014-07-01 16:23:13,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: expect (ObjNum 31 snap 0 seq_num 31) 2014-07-01 16:23:13,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 427 16 2278 2262 21.1856 52 1.05432 3.01552 2014-07-01 16:23:13,524.524 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:23:13,539.539 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:23:14,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:877: expect (ObjNum 280 snap 0 seq_num 280) 2014-07-01 16:23:14,342.342 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 428 16 2284 2268 21.1922 24 1.87439 3.01101 2014-07-01 16:23:14,538.538 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:23:14,549.549 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:23:15,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 429 16 2284 2268 21.1428 0 - 3.01101 2014-07-01 16:23:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: finishing write tid 1 to mira0435601-348 2014-07-01 16:23:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: finishing write tid 3 to mira0435601-348 2014-07-01 16:23:15,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: finishing write tid 4 to mira0435601-348 2014-07-01 16:23:15,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 140 (ObjNum 706 snap 0 seq_num 706) dirty exists 2014-07-01 16:23:15,564.564 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:23:15,576.576 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:23:15,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 1 to mira0435601-341 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 3 to mira0435601-341 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: finishing write tid 1 to mira0435601-191 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: finishing write tid 2 to mira0435601-191 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: finishing write tid 4 to mira0435601-191 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: done (14 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: done (13 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: done (12 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: done (11 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:877: done (10 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: done (9 left) 2014-07-01 16:23:15,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: done (8 left) 2014-07-01 16:23:15,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: done (7 left) 2014-07-01 16:23:15,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: done (6 left) 2014-07-01 16:23:15,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: done (5 left) 2014-07-01 16:23:15,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: done (4 left) 2014-07-01 16:23:15,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: done (3 left) 2014-07-01 16:23:15,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: read oid 130 snap -1 2014-07-01 16:23:15,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 1 to mira0435601-498 2014-07-01 16:23:15,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 2 to mira0435601-498 2014-07-01 16:23:15,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 168 current snap is 0 2014-07-01 16:23:15,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: seq_num 711 ranges {2211840=655360,2867200=700416,3567616=503808} 2014-07-01 16:23:15,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: writing mira0435601-168 from 2211840 to 2867200 tid 1 2014-07-01 16:23:15,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: writing mira0435601-168 from 2867200 to 3567616 tid 2 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: writing mira0435601-168 from 3567616 to 4071424 tid 3 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 408 current snap is 0 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: seq_num 712 ranges {1032192=684032} 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: writing mira0435601-408 from 1032192 to 1716224 tid 1 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:897: delete oid 104 current snap is 0 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 3 to mira0435601-498 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 4 to mira0435601-498 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 6 to mira0435601-498 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 4 to mira0435601-341 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 123 (ObjNum 707 snap 0 seq_num 707) dirty exists 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: finishing write tid 5 to mira0435601-191 2014-07-01 16:23:15,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 141 (ObjNum 708 snap 0 seq_num 708) dirty exists 2014-07-01 16:23:15,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: expect (ObjNum 129 snap 0 seq_num 129) 2014-07-01 16:23:16,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: finishing write tid 7 to mira0435601-498 2014-07-01 16:23:16,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 145 (ObjNum 709 snap 0 seq_num 709) dirty exists 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 1 to mira0435601-168 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 2 to mira0435601-168 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 3 to mira0435601-168 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 5 to mira0435601-168 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 6 to mira0435601-168 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 128 (ObjNum 711 snap 0 seq_num 711) dirty exists 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: done (6 left) 2014-07-01 16:23:16,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: done (5 left) 2014-07-01 16:23:16,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: done (4 left) 2014-07-01 16:23:16,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: done (3 left) 2014-07-01 16:23:16,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: done (2 left) 2014-07-01 16:23:16,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:897: done (1 left) 2014-07-01 16:23:16,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:898: read oid 398 snap -1 2014-07-01 16:23:16,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: read oid 246 snap -1 2014-07-01 16:23:16,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:900: read oid 318 snap -1 2014-07-01 16:23:16,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: read oid 76 snap -1 2014-07-01 16:23:16,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 294 current snap is 0 2014-07-01 16:23:16,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: seq_num 714 ranges {385024=229376} 2014-07-01 16:23:16,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: writing mira0435601-294 from 385024 to 614400 tid 1 2014-07-01 16:23:16,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 451 current snap is 0 2014-07-01 16:23:16,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: seq_num 715 ranges {540672=102400} 2014-07-01 16:23:16,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: writing mira0435601-451 from 540672 to 643072 tid 1 2014-07-01 16:23:16,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 335 current snap is 0 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: seq_num 716 ranges {3219456=774144,3993600=667648,4661248=479232,5140480=458752} 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0435601-335 from 3219456 to 3993600 tid 1 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0435601-335 from 3993600 to 4661248 tid 2 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0435601-335 from 4661248 to 5140480 tid 3 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0435601-335 from 5140480 to 5599232 tid 4 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: seq_num 717 ranges {249856=450560,700416=602112,1302528=753664,2056192=249856} 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: writing mira0435601-176 from 249856 to 700416 tid 1 2014-07-01 16:23:16,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: writing mira0435601-176 from 700416 to 1302528 tid 2 2014-07-01 16:23:16,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 430 16 2303 2287 21.2703 38 0.523127 3.0061 2014-07-01 16:23:16,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: writing mira0435601-176 from 1302528 to 2056192 tid 3 2014-07-01 16:23:16,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: writing mira0435601-176 from 2056192 to 2306048 tid 4 2014-07-01 16:23:16,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:906: read oid 463 snap -1 2014-07-01 16:23:16,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: expect (ObjNum 245 snap 0 seq_num 245) 2014-07-01 16:23:16,579.579 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:23:16,589.589 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:23:17,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: expect (ObjNum 75 snap 0 seq_num 75) 2014-07-01 16:23:17,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 431 16 2317 2301 21.3509 56 0.989096 2.99312 2014-07-01 16:23:17,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:898: expect (ObjNum 397 snap 0 seq_num 397) 2014-07-01 16:23:17,597.597 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:23:17,609.609 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:23:18,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 432 16 2319 2303 21.32 8 1.47869 2.9918 2014-07-01 16:23:18,810.810 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:23:18,821.821 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:23:18,821.821 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:23:19,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:906: expect (ObjNum 462 snap 0 seq_num 462) 2014-07-01 16:23:19,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 433 16 2325 2309 21.3262 24 2.27869 2.98999 2014-07-01 16:23:19,530.530 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:23:19,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:900: expect (ObjNum 317 snap 0 seq_num 317) 2014-07-01 16:23:20,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 434 16 2328 2312 21.3047 12 2.80419 2.98976 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 1 to mira0435601-408 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 3 to mira0435601-408 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 4 to mira0435601-408 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 139 (ObjNum 712 snap 0 seq_num 712) dirty exists 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 1 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 2 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 3 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 4 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 6 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: finishing write tid 7 to mira0435601-176 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 150 (ObjNum 717 snap 0 seq_num 717) dirty exists 2014-07-01 16:23:20,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: finishing write tid 1 to mira0435601-451 2014-07-01 16:23:20,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: finishing write tid 3 to mira0435601-451 2014-07-01 16:23:20,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 1 to mira0435601-294 2014-07-01 16:23:20,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 3 to mira0435601-294 2014-07-01 16:23:20,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: finishing write tid 4 to mira0435601-451 2014-07-01 16:23:20,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 128 (ObjNum 715 snap 0 seq_num 715) dirty exists 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 4 to mira0435601-294 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 294 v 195 (ObjNum 714 snap 0 seq_num 714) dirty exists 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 1 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 2 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 3 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 4 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 6 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 7 to mira0435601-335 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 120 (ObjNum 716 snap 0 seq_num 716) dirty exists 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: done (9 left) 2014-07-01 16:23:20,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:898: done (8 left) 2014-07-01 16:23:20,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: done (7 left) 2014-07-01 16:23:20,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:900: done (6 left) 2014-07-01 16:23:20,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: done (5 left) 2014-07-01 16:23:20,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: done (4 left) 2014-07-01 16:23:20,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: done (3 left) 2014-07-01 16:23:20,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: done (2 left) 2014-07-01 16:23:20,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: done (1 left) 2014-07-01 16:23:20,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:906: done (0 left) 2014-07-01 16:23:20,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 295 current snap is 0 2014-07-01 16:23:20,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: seq_num 718 ranges {1110016=569344} 2014-07-01 16:23:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: writing mira0435601-295 from 1110016 to 1679360 tid 1 2014-07-01 16:23:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:908: delete oid 340 current snap is 0 2014-07-01 16:23:20,892.892 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:23:21,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 435 16 2343 2327 21.3936 60 0.635104 2.98589 2014-07-01 16:23:21,623.623 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:23:22,343.343 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 436 16 2344 2328 21.3537 4 0.827238 2.98497 2014-07-01 16:23:22,653.653 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:23:23,344.344 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 437 16 2350 2334 21.3598 24 3.69182 2.98455 2014-07-01 16:23:23,389.389 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 10, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 10, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008094', u'skew': u'-0.018305', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002277', u'skew': u'-0.019842', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:22:35.053562', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884262324, u'health': u'HEALTH_OK', u'kb_used': 12518728, u'store_stats': {u'bytes_total': 9730999, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1342375}}, {u'last_updated': u'2014-07-01 07:22:54.528896', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887509204, u'health': u'HEALTH_OK', u'kb_used': 9271848, u'store_stats': {u'bytes_total': 2886988, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1838396}}, {u'last_updated': u'2014-07-01 07:23:03.112274', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887499364, u'health': u'HEALTH_OK', u'kb_used': 9281688, u'store_stats': {u'bytes_total': 3935583, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1838415}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 44 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 56 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 174, u'data_bytes': 10470104517, u'bytes_used': 22032166912, u'op_per_sec': 3, u'version': 375, u'pgs_by_state': [{u'count': 174, u'state_name': u'active+clean'}], u'read_bytes_sec': 452, u'write_bytes_sec': 8234009, u'bytes_avail': 5976266878976}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 181, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:23:23,389.389 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:23:23,389.389 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:23:23,389.389 INFO:teuthology.task.thrashosds.thrasher:Removing osd 5, in_osds are: [4, 5, 2, 1, 0, 3] 2014-07-01 16:23:23,389.389 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 5' 2014-07-01 16:23:23,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: finishing write tid 1 to mira0435601-295 2014-07-01 16:23:23,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: finishing write tid 3 to mira0435601-295 2014-07-01 16:23:23,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:908: done (1 left) 2014-07-01 16:23:23,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: read oid 483 snap -1 2014-07-01 16:23:23,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 4 current snap is 0 2014-07-01 16:23:23,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: seq_num 720 ranges {0=622592} 2014-07-01 16:23:23,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: writing mira0435601-4 from 0 to 622592 tid 1 2014-07-01 16:23:23,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 346 current snap is 0 2014-07-01 16:23:23,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: seq_num 721 ranges {868352=741376,1609728=425984,2035712=376832} 2014-07-01 16:23:23,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: writing mira0435601-346 from 868352 to 1609728 tid 1 2014-07-01 16:23:23,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: writing mira0435601-346 from 1609728 to 2035712 tid 2 2014-07-01 16:23:23,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: writing mira0435601-346 from 2035712 to 2412544 tid 3 2014-07-01 16:23:23,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 204 current snap is 0 2014-07-01 16:23:23,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: seq_num 722 ranges {409600=413696,823296=659456,1482752=143360} 2014-07-01 16:23:23,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: writing mira0435601-204 from 409600 to 823296 tid 1 2014-07-01 16:23:23,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: writing mira0435601-204 from 823296 to 1482752 tid 2 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: writing mira0435601-204 from 1482752 to 1626112 tid 3 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:913: read oid 166 snap -1 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 54 current snap is 0 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: seq_num 723 ranges {1470464=40960} 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: writing mira0435601-54 from 1470464 to 1511424 tid 1 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 403 current snap is 0 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: seq_num 724 ranges {0=716800,716800=471040,1187840=421888,1609728=536576,2146304=176128} 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0435601-403 from 0 to 716800 tid 1 2014-07-01 16:23:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0435601-403 from 716800 to 1187840 tid 2 2014-07-01 16:23:23,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0435601-403 from 1187840 to 1609728 tid 3 2014-07-01 16:23:23,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0435601-403 from 1609728 to 2146304 tid 4 2014-07-01 16:23:23,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0435601-403 from 2146304 to 2322432 tid 5 2014-07-01 16:23:23,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 103 current snap is 0 2014-07-01 16:23:23,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: seq_num 725 ranges {2904064=585728,3489792=409600,3899392=647168,4546560=548864} 2014-07-01 16:23:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: writing mira0435601-103 from 2904064 to 3489792 tid 1 2014-07-01 16:23:23,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: writing mira0435601-103 from 3489792 to 3899392 tid 2 2014-07-01 16:23:23,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: writing mira0435601-103 from 3899392 to 4546560 tid 3 2014-07-01 16:23:23,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: writing mira0435601-103 from 4546560 to 5095424 tid 4 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 338 current snap is 0 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: seq_num 726 ranges {1978368=663552,2641920=638976,3280896=475136,3756032=90112} 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: writing mira0435601-338 from 1978368 to 2641920 tid 1 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: writing mira0435601-338 from 2641920 to 3280896 tid 2 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: writing mira0435601-338 from 3280896 to 3756032 tid 3 2014-07-01 16:23:24,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: writing mira0435601-338 from 3756032 to 3846144 tid 4 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: seq_num 727 ranges {2318336=536576,2854912=622592,3477504=581632} 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: writing mira0435601-128 from 2318336 to 2854912 tid 1 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: writing mira0435601-128 from 2854912 to 3477504 tid 2 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: writing mira0435601-128 from 3477504 to 4059136 tid 3 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 307 current snap is 0 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: seq_num 728 ranges {847872=405504,1253376=454656,1708032=421888,2129920=327680} 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: writing mira0435601-307 from 847872 to 1253376 tid 1 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: writing mira0435601-307 from 1253376 to 1708032 tid 2 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: writing mira0435601-307 from 1708032 to 2129920 tid 3 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: writing mira0435601-307 from 2129920 to 2457600 tid 4 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 95 current snap is 0 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: seq_num 729 ranges {0=434176} 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: writing mira0435601-95 from 0 to 434176 tid 1 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: delete oid 138 current snap is 0 2014-07-01 16:23:24,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: expect (ObjNum 482 snap 0 seq_num 482) 2014-07-01 16:23:24,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 438 16 2366 2350 21.4571 64 3.14694 2.97813 2014-07-01 16:23:25,139.139 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.5. 2014-07-01 16:23:25,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: finishing write tid 4 to mira0435601-295 2014-07-01 16:23:25,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 130 (ObjNum 718 snap 0 seq_num 718) dirty exists 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: finishing write tid 1 to mira0435601-204 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: finishing write tid 2 to mira0435601-204 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: finishing write tid 3 to mira0435601-204 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: finishing write tid 5 to mira0435601-204 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: finishing write tid 6 to mira0435601-204 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 145 (ObjNum 722 snap 0 seq_num 722) dirty exists 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 1 to mira0435601-103 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 2 to mira0435601-103 2014-07-01 16:23:25,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 3 to mira0435601-103 2014-07-01 16:23:25,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 4 to mira0435601-103 2014-07-01 16:23:25,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 6 to mira0435601-103 2014-07-01 16:23:25,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: finishing write tid 7 to mira0435601-103 2014-07-01 16:23:25,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 119 (ObjNum 725 snap 0 seq_num 725) dirty exists 2014-07-01 16:23:25,344.344 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 439 16 2368 2352 21.4264 8 1.00247 2.97646 2014-07-01 16:23:25,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 1 to mira0435601-4 2014-07-01 16:23:25,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 3 to mira0435601-4 2014-07-01 16:23:25,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: finishing write tid 1 to mira0435601-54 2014-07-01 16:23:25,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: finishing write tid 3 to mira0435601-54 2014-07-01 16:23:25,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 1 to mira0435601-403 2014-07-01 16:23:25,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: finishing write tid 4 to mira0435601-54 2014-07-01 16:23:25,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 116 (ObjNum 723 snap 0 seq_num 723) dirty exists 2014-07-01 16:23:25,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 4 to mira0435601-4 2014-07-01 16:23:25,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 130 (ObjNum 720 snap 0 seq_num 720) dirty exists 2014-07-01 16:23:26,344.344 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:23:26.258101min lat: 0.057269 max lat: 48.8206 avg lat: 2.9699 2014-07-01 16:23:26,344.344 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:23:26,345.345 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 440 16 2378 2362 21.4686 40 0.702606 2.9699 2014-07-01 16:23:27,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 441 16 2379 2363 21.429 4 1.06095 2.9691 2014-07-01 16:23:28,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 2 to mira0435601-403 2014-07-01 16:23:28,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 3 to mira0435601-403 2014-07-01 16:23:28,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 1 to mira0435601-338 2014-07-01 16:23:28,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 4 to mira0435601-403 2014-07-01 16:23:28,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 5 to mira0435601-403 2014-07-01 16:23:28,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 7 to mira0435601-403 2014-07-01 16:23:28,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 2 to mira0435601-338 2014-07-01 16:23:28,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 3 to mira0435601-338 2014-07-01 16:23:28,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 4 to mira0435601-338 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: finishing write tid 1 to mira0435601-128 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 6 to mira0435601-338 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: finishing write tid 2 to mira0435601-128 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: finishing write tid 3 to mira0435601-128 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: finishing write tid 5 to mira0435601-128 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: finishing write tid 6 to mira0435601-128 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 122 (ObjNum 727 snap 0 seq_num 727) dirty exists 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: finishing write tid 7 to mira0435601-338 2014-07-01 16:23:28,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 158 (ObjNum 726 snap 0 seq_num 726) dirty exists 2014-07-01 16:23:28,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 8 to mira0435601-403 2014-07-01 16:23:28,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 201 (ObjNum 724 snap 0 seq_num 724) dirty exists 2014-07-01 16:23:28,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 442 16 2385 2369 21.4348 24 1.6858 2.97088 2014-07-01 16:23:29,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 443 16 2396 2380 21.4857 44 0.37636 2.96304 2014-07-01 16:23:30,151.151 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:23:30,151.151 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:23:30,152.152 INFO:teuthology.task.thrashosds.thrasher:Growing pool data 2014-07-01 16:23:30,152.152 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:23:30,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 444 16 2396 2380 21.4373 0 - 2.96304 2014-07-01 16:23:30,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: finishing write tid 1 to mira0435601-346 2014-07-01 16:23:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: finishing write tid 2 to mira0435601-346 2014-07-01 16:23:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: finishing write tid 3 to mira0435601-346 2014-07-01 16:23:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: finishing write tid 5 to mira0435601-346 2014-07-01 16:23:30,724.724 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:23:30,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: finishing write tid 6 to mira0435601-346 2014-07-01 16:23:30,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 132 (ObjNum 721 snap 0 seq_num 721) dirty exists 2014-07-01 16:23:31,347.347 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 445 16 2403 2387 21.4521 14 1.81834 2.9603 2014-07-01 16:23:32,095.095 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:23:32,095.095 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pg_num 34' 2014-07-01 16:23:32,346.346 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 446 16 2409 2393 21.4578 24 6.4147 2.9655 2014-07-01 16:23:33,348.348 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 447 16 2415 2399 21.4634 24 1.88833 2.96205 2014-07-01 16:23:33,664.664 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pg_num to 34 2014-07-01 16:23:34,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 448 16 2428 2412 21.5316 52 1.72906 2.95484 2014-07-01 16:23:35,348.348 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 449 16 2428 2412 21.4836 0 - 2.95484 2014-07-01 16:23:36,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: done (13 left) 2014-07-01 16:23:36,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: done (12 left) 2014-07-01 16:23:36,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: done (11 left) 2014-07-01 16:23:36,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: done (10 left) 2014-07-01 16:23:36,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: done (9 left) 2014-07-01 16:23:36,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:913: done (8 left) 2014-07-01 16:23:36,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: done (7 left) 2014-07-01 16:23:36,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: done (6 left) 2014-07-01 16:23:36,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: done (5 left) 2014-07-01 16:23:36,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: done (4 left) 2014-07-01 16:23:36,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: done (3 left) 2014-07-01 16:23:36,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: done (2 left) 2014-07-01 16:23:36,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:922: read oid 426 snap -1 2014-07-01 16:23:36,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 473 current snap is 0 2014-07-01 16:23:36,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: seq_num 731 ranges {815104=626688,1441792=401408} 2014-07-01 16:23:36,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: writing mira0435601-473 from 815104 to 1441792 tid 1 2014-07-01 16:23:36,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: writing mira0435601-473 from 1441792 to 1843200 tid 2 2014-07-01 16:23:36,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 120 current snap is 0 2014-07-01 16:23:36,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: seq_num 732 ranges {1331200=655360,1986560=679936,2666496=712704,3379200=212992} 2014-07-01 16:23:36,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: writing mira0435601-120 from 1331200 to 1986560 tid 1 2014-07-01 16:23:36,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: writing mira0435601-120 from 1986560 to 2666496 tid 2 2014-07-01 16:23:36,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: writing mira0435601-120 from 2666496 to 3379200 tid 3 2014-07-01 16:23:36,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: writing mira0435601-120 from 3379200 to 3592192 tid 4 2014-07-01 16:23:36,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: delete oid 494 current snap is 0 2014-07-01 16:23:36,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: finishing write tid 1 to mira0435601-95 2014-07-01 16:23:36,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: finishing write tid 3 to mira0435601-95 2014-07-01 16:23:36,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:922: expect (ObjNum 425 snap 0 seq_num 425) 2014-07-01 16:23:36,347.347 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 450 16 2431 2415 21.4625 6 1.6803 2.95435 2014-07-01 16:23:37,348.348 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 451 16 2432 2416 21.4238 4 13.0982 2.95855 2014-07-01 16:23:38,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 452 16 2432 2416 21.3764 0 - 2.95855 2014-07-01 16:23:38,678.678 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:23:38,679.679 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:23:38,679.679 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 16:23:38,680.680 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:23:39,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 453 16 2440 2424 21.3999 16 14.8728 2.97427 2014-07-01 16:23:39,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:922: done (5 left) 2014-07-01 16:23:39,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: done (4 left) 2014-07-01 16:23:39,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 16:23:39,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: seq_num 734 ranges {4059136=798720,4857856=442368,5300224=659456,5959680=102400} 2014-07-01 16:23:39,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: writing mira0435601-128 from 4059136 to 4857856 tid 1 2014-07-01 16:23:39,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: writing mira0435601-128 from 4857856 to 5300224 tid 2 2014-07-01 16:23:39,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: writing mira0435601-128 from 5300224 to 5959680 tid 3 2014-07-01 16:23:39,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: writing mira0435601-128 from 5959680 to 6062080 tid 4 2014-07-01 16:23:39,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:927: read oid 184 snap -1 2014-07-01 16:23:39,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: finishing write tid 1 to mira0435601-473 2014-07-01 16:23:39,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: finishing write tid 2 to mira0435601-473 2014-07-01 16:23:39,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: finishing write tid 4 to mira0435601-473 2014-07-01 16:23:39,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: finishing write tid 5 to mira0435601-473 2014-07-01 16:23:39,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 148 (ObjNum 731 snap 0 seq_num 731) dirty exists 2014-07-01 16:23:39,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: done (5 left) 2014-07-01 16:23:39,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 469 current snap is 0 2014-07-01 16:23:39,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: seq_num 735 ranges {1523712=475136,1998848=430080,2428928=487424,2916352=286720} 2014-07-01 16:23:40,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0435601-469 from 1523712 to 1998848 tid 1 2014-07-01 16:23:40,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0435601-469 from 1998848 to 2428928 tid 2 2014-07-01 16:23:40,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0435601-469 from 2428928 to 2916352 tid 3 2014-07-01 16:23:40,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0435601-469 from 2916352 to 3203072 tid 4 2014-07-01 16:23:40,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 286 current snap is 0 2014-07-01 16:23:40,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: seq_num 736 ranges {1298432=749568,2048000=659456,2707456=143360} 2014-07-01 16:23:40,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: writing mira0435601-286 from 1298432 to 2048000 tid 1 2014-07-01 16:23:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: writing mira0435601-286 from 2048000 to 2707456 tid 2 2014-07-01 16:23:40,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: writing mira0435601-286 from 2707456 to 2850816 tid 3 2014-07-01 16:23:40,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:930: read oid 132 snap -1 2014-07-01 16:23:40,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: read oid 81 snap -1 2014-07-01 16:23:40,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:932: read oid 448 snap -1 2014-07-01 16:23:40,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:933: read oid 192 snap -1 2014-07-01 16:23:40,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: delete oid 21 current snap is 0 2014-07-01 16:23:40,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:930: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 16:23:40,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 454 16 2441 2425 21.3615 4 3.84657 2.97463 2014-07-01 16:23:40,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:930: done (11 left) 2014-07-01 16:23:40,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: done (10 left) 2014-07-01 16:23:40,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 410 current snap is 0 2014-07-01 16:23:40,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: seq_num 738 ranges {778240=245760} 2014-07-01 16:23:40,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: writing mira0435601-410 from 778240 to 1024000 tid 1 2014-07-01 16:23:40,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 490 current snap is 0 2014-07-01 16:23:40,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: seq_num 739 ranges {1925120=114688} 2014-07-01 16:23:40,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: writing mira0435601-490 from 1925120 to 2039808 tid 1 2014-07-01 16:23:40,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:937: read oid 350 snap -1 2014-07-01 16:23:40,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:938: read oid 149 snap -1 2014-07-01 16:23:40,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 16:23:40,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: seq_num 740 ranges {417792=593920,1011712=483328,1495040=606208,2101248=516096} 2014-07-01 16:23:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: writing mira0435601-238 from 417792 to 1011712 tid 1 2014-07-01 16:23:40,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: writing mira0435601-238 from 1011712 to 1495040 tid 2 2014-07-01 16:23:40,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: writing mira0435601-238 from 1495040 to 2101248 tid 3 2014-07-01 16:23:40,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: writing mira0435601-238 from 2101248 to 2617344 tid 4 2014-07-01 16:23:40,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: read oid 6 snap -1 2014-07-01 16:23:40,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:40,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:932: expect (ObjNum 621 snap 0 seq_num 621) 2014-07-01 16:23:41,075.075 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:23:41,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 455 16 2446 2430 21.3585 20 6.29711 2.98048 2014-07-01 16:23:42,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 456 16 2450 2434 21.3468 16 0.15046 2.98194 2014-07-01 16:23:43,349.349 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 457 16 2455 2439 21.3438 20 3.8279 2.98234 2014-07-01 16:23:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 1 to mira0435601-238 2014-07-01 16:23:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 2 to mira0435601-238 2014-07-01 16:23:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 3 to mira0435601-238 2014-07-01 16:23:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 4 to mira0435601-238 2014-07-01 16:23:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 6 to mira0435601-238 2014-07-01 16:23:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: finishing write tid 7 to mira0435601-238 2014-07-01 16:23:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 139 (ObjNum 740 snap 0 seq_num 740) dirty exists 2014-07-01 16:23:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 1 to mira0435601-120 2014-07-01 16:23:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:932: done (15 left) 2014-07-01 16:23:43,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: done (14 left) 2014-07-01 16:23:43,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 16:23:43,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: seq_num 741 ranges {2617344=466944,3084288=421888,3506176=483328,3989504=618496,4608000=81920} 2014-07-01 16:23:43,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: writing mira0435601-238 from 2617344 to 3084288 tid 1 2014-07-01 16:23:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: writing mira0435601-238 from 3084288 to 3506176 tid 2 2014-07-01 16:23:43,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: writing mira0435601-238 from 3506176 to 3989504 tid 3 2014-07-01 16:23:43,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: writing mira0435601-238 from 3989504 to 4608000 tid 4 2014-07-01 16:23:43,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: writing mira0435601-238 from 4608000 to 4689920 tid 5 2014-07-01 16:23:44,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 242 current snap is 0 2014-07-01 16:23:44,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: seq_num 742 ranges {753664=737280,1490944=651264,2142208=552960} 2014-07-01 16:23:44,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: writing mira0435601-242 from 753664 to 1490944 tid 1 2014-07-01 16:23:44,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: writing mira0435601-242 from 1490944 to 2142208 tid 2 2014-07-01 16:23:44,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: writing mira0435601-242 from 2142208 to 2695168 tid 3 2014-07-01 16:23:44,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:23:44,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 2 to mira0435601-120 2014-07-01 16:23:44,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 3 to mira0435601-120 2014-07-01 16:23:44,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 4 to mira0435601-120 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 6 to mira0435601-120 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 1 to mira0435601-128 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 2 to mira0435601-128 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 1 to mira0435601-307 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 2 to mira0435601-307 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 3 to mira0435601-307 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 4 to mira0435601-307 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 6 to mira0435601-307 2014-07-01 16:23:44,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:937: expect (ObjNum 349 snap 0 seq_num 349) 2014-07-01 16:23:44,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 458 16 2458 2442 21.3234 12 4.74677 2.98375 2014-07-01 16:23:44,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 7 to mira0435601-120 2014-07-01 16:23:44,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 79 (ObjNum 732 snap 0 seq_num 732) dirty exists 2014-07-01 16:23:44,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: finishing write tid 1 to mira0435601-286 2014-07-01 16:23:44,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: finishing write tid 2 to mira0435601-286 2014-07-01 16:23:44,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: finishing write tid 4 to mira0435601-95 2014-07-01 16:23:44,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 152 (ObjNum 729 snap 0 seq_num 729) dirty exists 2014-07-01 16:23:44,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: expect (ObjNum 528 snap 0 seq_num 528) 2014-07-01 16:23:45,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:927: expect (ObjNum 183 snap 0 seq_num 183) 2014-07-01 16:23:45,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 459 16 2468 2452 21.3641 40 4.67585 2.98921 2014-07-01 16:23:45,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: expect (ObjNum 80 snap 0 seq_num 80) 2014-07-01 16:23:46,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:23:46.262263min lat: 0.057269 max lat: 48.8206 avg lat: 2.98419 2014-07-01 16:23:46,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:23:46,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 460 16 2478 2462 21.4046 40 2.40715 2.98419 2014-07-01 16:23:46,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: finishing write tid 3 to mira0435601-286 2014-07-01 16:23:46,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: finishing write tid 5 to mira0435601-286 2014-07-01 16:23:46,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: finishing write tid 7 to mira0435601-307 2014-07-01 16:23:46,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 133 (ObjNum 728 snap 0 seq_num 728) dirty exists 2014-07-01 16:23:46,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: done (15 left) 2014-07-01 16:23:46,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: done (14 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: done (13 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:927: done (12 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: done (11 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:937: done (10 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: done (9 left) 2014-07-01 16:23:46,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 324 current snap is 0 2014-07-01 16:23:46,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: seq_num 743 ranges {1646592=765952} 2014-07-01 16:23:46,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: writing mira0435601-324 from 1646592 to 2412544 tid 1 2014-07-01 16:23:46,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: read oid 203 snap -1 2014-07-01 16:23:46,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: finishing write tid 6 to mira0435601-286 2014-07-01 16:23:46,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 205 (ObjNum 736 snap 0 seq_num 736) dirty exists 2014-07-01 16:23:46,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:938: expect (ObjNum 148 snap 0 seq_num 148) 2014-07-01 16:23:47,269.269 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:23:47,270.270 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:23:47,271.271 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 16:23:47,272.272 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:23:47,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 461 16 2478 2462 21.3582 0 - 2.98419 2014-07-01 16:23:47,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 3 to mira0435601-128 2014-07-01 16:23:47,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 4 to mira0435601-128 2014-07-01 16:23:47,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 6 to mira0435601-128 2014-07-01 16:23:47,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 1 to mira0435601-238 2014-07-01 16:23:47,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 2 to mira0435601-238 2014-07-01 16:23:47,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 3 to mira0435601-238 2014-07-01 16:23:47,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 4 to mira0435601-238 2014-07-01 16:23:47,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 5 to mira0435601-238 2014-07-01 16:23:47,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 7 to mira0435601-238 2014-07-01 16:23:47,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: finishing write tid 8 to mira0435601-238 2014-07-01 16:23:47,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 145 (ObjNum 741 snap 0 seq_num 741) dirty exists 2014-07-01 16:23:47,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: finishing write tid 1 to mira0435601-410 2014-07-01 16:23:47,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: done (10 left) 2014-07-01 16:23:47,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:938: done (9 left) 2014-07-01 16:23:47,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: done (8 left) 2014-07-01 16:23:47,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 197 current snap is 0 2014-07-01 16:23:47,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: seq_num 744 ranges {1699840=688128,2387968=466944,2854912=335872} 2014-07-01 16:23:47,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: writing mira0435601-197 from 1699840 to 2387968 tid 1 2014-07-01 16:23:47,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: writing mira0435601-197 from 2387968 to 2854912 tid 2 2014-07-01 16:23:47,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: writing mira0435601-197 from 2854912 to 3190784 tid 3 2014-07-01 16:23:47,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 178 current snap is 0 2014-07-01 16:23:47,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: seq_num 745 ranges {847872=557056,1404928=802816} 2014-07-01 16:23:47,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: writing mira0435601-178 from 847872 to 1404928 tid 1 2014-07-01 16:23:47,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: writing mira0435601-178 from 1404928 to 2207744 tid 2 2014-07-01 16:23:47,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: delete oid 303 current snap is 0 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: finishing write tid 3 to mira0435601-410 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: finishing write tid 4 to mira0435601-410 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 207 (ObjNum 738 snap 0 seq_num 738) dirty exists 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: finishing write tid 7 to mira0435601-128 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 127 (ObjNum 734 snap 0 seq_num 734) dirty exists 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 1 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 2 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 3 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 4 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 6 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 7 to mira0435601-469 2014-07-01 16:23:47,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 84 (ObjNum 735 snap 0 seq_num 735) dirty exists 2014-07-01 16:23:48,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 462 16 2478 2462 21.3119 0 - 2.98419 2014-07-01 16:23:48,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 1 to mira0435601-490 2014-07-01 16:23:48,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 3 to mira0435601-490 2014-07-01 16:23:49,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 4 to mira0435601-490 2014-07-01 16:23:49,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 86 (ObjNum 739 snap 0 seq_num 739) dirty exists 2014-07-01 16:23:49,109.109 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:23:49,352.352 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 463 16 2482 2466 21.3005 5.33333 3.74626 2.98497 2014-07-01 16:23:50,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 464 16 2482 2466 21.2545 0 - 2.98497 2014-07-01 16:23:50,493.493 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 24' 2014-07-01 16:23:50,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: done (10 left) 2014-07-01 16:23:50,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: done (9 left) 2014-07-01 16:23:50,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: done (8 left) 2014-07-01 16:23:50,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: done (7 left) 2014-07-01 16:23:50,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: done (6 left) 2014-07-01 16:23:50,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: done (5 left) 2014-07-01 16:23:50,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 405 current snap is 0 2014-07-01 16:23:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: seq_num 747 ranges {1613824=57344} 2014-07-01 16:23:50,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: writing mira0435601-405 from 1613824 to 1671168 tid 1 2014-07-01 16:23:50,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:949: read oid 83 snap -1 2014-07-01 16:23:50,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 1 to mira0435601-178 2014-07-01 16:23:50,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 2 to mira0435601-178 2014-07-01 16:23:50,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 4 to mira0435601-178 2014-07-01 16:23:50,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:950: delete oid 63 current snap is 0 2014-07-01 16:23:50,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 5 to mira0435601-178 2014-07-01 16:23:50,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 167 (ObjNum 745 snap 0 seq_num 745) dirty exists 2014-07-01 16:23:50,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:949: expect (ObjNum 82 snap 0 seq_num 82) 2014-07-01 16:23:51,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 465 16 2483 2467 21.2174 2 1.70318 2.98446 2014-07-01 16:23:51,507.507 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 24 2014-07-01 16:23:52,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 1 to mira0435601-242 2014-07-01 16:23:52,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 2 to mira0435601-242 2014-07-01 16:23:52,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: done (7 left) 2014-07-01 16:23:52,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:949: done (6 left) 2014-07-01 16:23:52,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:950: done (5 left) 2014-07-01 16:23:52,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 475 current snap is 0 2014-07-01 16:23:52,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: seq_num 749 ranges {786432=405504,1191936=782336,1974272=774144,2748416=49152} 2014-07-01 16:23:52,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: writing mira0435601-475 from 786432 to 1191936 tid 1 2014-07-01 16:23:52,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: writing mira0435601-475 from 1191936 to 1974272 tid 2 2014-07-01 16:23:52,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: writing mira0435601-475 from 1974272 to 2748416 tid 3 2014-07-01 16:23:52,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: writing mira0435601-475 from 2748416 to 2797568 tid 4 2014-07-01 16:23:52,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:952: read oid 243 snap -1 2014-07-01 16:23:52,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 3 to mira0435601-242 2014-07-01 16:23:52,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 388 current snap is 0 2014-07-01 16:23:52,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: seq_num 750 ranges {0=434176,434176=401408} 2014-07-01 16:23:52,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: writing mira0435601-388 from 0 to 434176 tid 1 2014-07-01 16:23:52,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: writing mira0435601-388 from 434176 to 835584 tid 2 2014-07-01 16:23:52,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 464 current snap is 0 2014-07-01 16:23:52,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: seq_num 751 ranges {2105344=548864,2654208=442368,3096576=475136,3571712=286720} 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: writing mira0435601-464 from 2105344 to 2654208 tid 1 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: writing mira0435601-464 from 2654208 to 3096576 tid 2 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: writing mira0435601-464 from 3096576 to 3571712 tid 3 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: writing mira0435601-464 from 3571712 to 3858432 tid 4 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:955: delete oid 123 current snap is 0 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 5 to mira0435601-242 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 1 to mira0435601-324 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 3 to mira0435601-324 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 4 to mira0435601-324 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 134 (ObjNum 743 snap 0 seq_num 743) dirty exists 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 1 to mira0435601-405 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 3 to mira0435601-405 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 6 to mira0435601-242 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 113 (ObjNum 742 snap 0 seq_num 742) dirty exists 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 4 to mira0435601-405 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 160 (ObjNum 747 snap 0 seq_num 747) dirty exists 2014-07-01 16:23:52,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:933: expect (ObjNum 640 snap 0 seq_num 640) 2014-07-01 16:23:52,350.350 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 466 16 2504 2488 21.3521 84 0.52463 2.99209 2014-07-01 16:23:53,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 467 16 2507 2491 21.3321 12 1.42202 2.99023 2014-07-01 16:23:54,352.352 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 468 16 2509 2493 21.3036 8 2.14403 2.98948 2014-07-01 16:23:54,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:952: expect (ObjNum 559 snap 0 seq_num 559) 2014-07-01 16:23:55,352.352 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 469 16 2509 2493 21.2582 0 - 2.98948 2014-07-01 16:23:55,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 1 to mira0435601-475 2014-07-01 16:23:55,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 2 to mira0435601-475 2014-07-01 16:23:55,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 3 to mira0435601-475 2014-07-01 16:23:55,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 4 to mira0435601-475 2014-07-01 16:23:55,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 6 to mira0435601-475 2014-07-01 16:23:55,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: finishing write tid 7 to mira0435601-475 2014-07-01 16:23:55,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 156 (ObjNum 749 snap 0 seq_num 749) dirty exists 2014-07-01 16:23:56,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 470 16 2509 2493 21.213 0 - 2.98948 2014-07-01 16:23:56,515.515 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:23:56,516.516 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:23:56,516.516 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.141565 2014-07-01 16:23:56,517.517 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 0.141564531327' 2014-07-01 16:23:56,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 1 to mira0435601-464 2014-07-01 16:23:56,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 2 to mira0435601-464 2014-07-01 16:23:56,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: finishing write tid 1 to mira0435601-388 2014-07-01 16:23:56,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 3 to mira0435601-464 2014-07-01 16:23:56,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: finishing write tid 2 to mira0435601-388 2014-07-01 16:23:56,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 4 to mira0435601-464 2014-07-01 16:23:56,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: finishing write tid 4 to mira0435601-388 2014-07-01 16:23:56,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 6 to mira0435601-464 2014-07-01 16:23:56,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: finishing write tid 7 to mira0435601-464 2014-07-01 16:23:56,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 212 (ObjNum 751 snap 0 seq_num 751) dirty exists 2014-07-01 16:23:57,352.352 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 471 16 2515 2499 21.2189 8 0.157656 2.98662 2014-07-01 16:23:57,633.633 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 0.141565 (892772) 2014-07-01 16:23:58,353.353 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 472 16 2515 2499 21.1739 0 - 2.98662 2014-07-01 16:23:59,353.353 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 473 16 2515 2499 21.1291 0 - 2.98662 2014-07-01 16:23:59,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 1 to mira0435601-197 2014-07-01 16:23:59,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 2 to mira0435601-197 2014-07-01 16:23:59,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 3 to mira0435601-197 2014-07-01 16:23:59,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 5 to mira0435601-197 2014-07-01 16:23:59,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: finishing write tid 5 to mira0435601-388 2014-07-01 16:23:59,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 136 (ObjNum 750 snap 0 seq_num 750) dirty exists 2014-07-01 16:23:59,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 6 to mira0435601-197 2014-07-01 16:23:59,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 197 v 156 (ObjNum 744 snap 0 seq_num 744) dirty exists 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:933: done (9 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: done (8 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: done (7 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: done (6 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: done (5 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: done (4 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:952: done (3 left) 2014-07-01 16:23:59,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: done (2 left) 2014-07-01 16:23:59,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: done (1 left) 2014-07-01 16:23:59,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:955: done (0 left) 2014-07-01 16:23:59,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 116 current snap is 0 2014-07-01 16:23:59,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: seq_num 753 ranges {679936=253952} 2014-07-01 16:23:59,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: writing mira0435601-116 from 679936 to 933888 tid 1 2014-07-01 16:23:59,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: read oid 429 snap -1 2014-07-01 16:23:59,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 355 current snap is 0 2014-07-01 16:23:59,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: seq_num 754 ranges {679936=770048,1449984=401408} 2014-07-01 16:23:59,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: writing mira0435601-355 from 679936 to 1449984 tid 1 2014-07-01 16:23:59,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: writing mira0435601-355 from 1449984 to 1851392 tid 2 2014-07-01 16:23:59,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 489 current snap is 0 2014-07-01 16:23:59,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: seq_num 755 ranges {2400256=798720,3198976=421888,3620864=720896,4341760=430080,4771840=32768} 2014-07-01 16:23:59,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0435601-489 from 2400256 to 3198976 tid 1 2014-07-01 16:23:59,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0435601-489 from 3198976 to 3620864 tid 2 2014-07-01 16:23:59,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0435601-489 from 3620864 to 4341760 tid 3 2014-07-01 16:23:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0435601-489 from 4341760 to 4771840 tid 4 2014-07-01 16:23:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0435601-489 from 4771840 to 4804608 tid 5 2014-07-01 16:23:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:960: read oid 120 snap -1 2014-07-01 16:23:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 140 current snap is 0 2014-07-01 16:23:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: seq_num 756 ranges {2056192=425984,2482176=651264,3133440=151552} 2014-07-01 16:23:59,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: writing mira0435601-140 from 2056192 to 2482176 tid 1 2014-07-01 16:23:59,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: writing mira0435601-140 from 2482176 to 3133440 tid 2 2014-07-01 16:23:59,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: writing mira0435601-140 from 3133440 to 3284992 tid 3 2014-07-01 16:23:59,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:962: read oid 187 snap -1 2014-07-01 16:23:59,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:963: read oid 67 snap -1 2014-07-01 16:23:59,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: expect (ObjNum 428 snap 0 seq_num 428) 2014-07-01 16:23:59,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:963: expect (ObjNum 66 snap 0 seq_num 66) 2014-07-01 16:24:00,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:962: expect (ObjNum 627 snap 0 seq_num 627) 2014-07-01 16:24:00,351.351 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 474 16 2525 2509 21.1689 13.3333 6.26863 3.00542 2014-07-01 16:24:01,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:960: expect (ObjNum 732 snap 0 seq_num 732) 2014-07-01 16:24:01,352.352 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 475 16 2525 2509 21.1244 0 - 3.00542 2014-07-01 16:24:02,353.353 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 476 16 2525 2509 21.08 0 - 3.00542 2014-07-01 16:24:02,655.655 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:02,656.656 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:02,656.656 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:24:02,656.656 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:24:02,657.657 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:24:02,657.657 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:24:03,353.353 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 477 16 2525 2509 21.0358 0 - 3.00542 2014-07-01 16:24:03,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: done (7 left) 2014-07-01 16:24:03,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:960: done (6 left) 2014-07-01 16:24:03,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:962: done (5 left) 2014-07-01 16:24:03,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:963: done (4 left) 2014-07-01 16:24:03,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: read oid 390 snap -1 2014-07-01 16:24:03,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:24:03,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: seq_num 757 ranges {4435968=540672,4976640=589824,5566464=421888,5988352=593920} 2014-07-01 16:24:03,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0435601-337 from 4435968 to 4976640 tid 1 2014-07-01 16:24:03,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0435601-337 from 4976640 to 5566464 tid 2 2014-07-01 16:24:03,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0435601-337 from 5566464 to 5988352 tid 3 2014-07-01 16:24:03,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0435601-337 from 5988352 to 6582272 tid 4 2014-07-01 16:24:03,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 414 current snap is 0 2014-07-01 16:24:03,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: seq_num 758 ranges {1425408=618496,2043904=417792,2461696=495616,2957312=290816} 2014-07-01 16:24:03,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: writing mira0435601-414 from 1425408 to 2043904 tid 1 2014-07-01 16:24:03,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: writing mira0435601-414 from 2043904 to 2461696 tid 2 2014-07-01 16:24:03,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: writing mira0435601-414 from 2461696 to 2957312 tid 3 2014-07-01 16:24:03,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: writing mira0435601-414 from 2957312 to 3248128 tid 4 2014-07-01 16:24:03,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 338 current snap is 0 2014-07-01 16:24:03,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: seq_num 759 ranges {3846144=487424,4333568=802816,5136384=401408} 2014-07-01 16:24:03,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: writing mira0435601-338 from 3846144 to 4333568 tid 1 2014-07-01 16:24:03,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: writing mira0435601-338 from 4333568 to 5136384 tid 2 2014-07-01 16:24:03,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: writing mira0435601-338 from 5136384 to 5537792 tid 3 2014-07-01 16:24:03,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: read oid 310 snap -1 2014-07-01 16:24:03,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 427 current snap is 0 2014-07-01 16:24:03,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: seq_num 760 ranges {528384=761856,1290240=634880,1925120=167936} 2014-07-01 16:24:03,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: writing mira0435601-427 from 528384 to 1290240 tid 1 2014-07-01 16:24:03,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: writing mira0435601-427 from 1290240 to 1925120 tid 2 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: writing mira0435601-427 from 1925120 to 2093056 tid 3 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: read oid 23 snap -1 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 457 current snap is 0 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: seq_num 761 ranges {737280=389120} 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: writing mira0435601-457 from 737280 to 1126400 tid 1 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: seq_num 762 ranges {724992=708608,1433600=401408} 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: writing mira0435601-323 from 724992 to 1433600 tid 1 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: writing mira0435601-323 from 1433600 to 1835008 tid 2 2014-07-01 16:24:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:973: delete oid 369 current snap is 0 2014-07-01 16:24:04,353.353 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 478 16 2525 2509 20.9918 0 - 3.00542 2014-07-01 16:24:05,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 479 16 2525 2509 20.948 0 - 3.00542 2014-07-01 16:24:06,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:24:06.266052min lat: 0.057269 max lat: 48.8206 avg lat: 3.00542 2014-07-01 16:24:06,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:24:06,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 480 16 2525 2509 20.9043 0 - 3.00542 2014-07-01 16:24:07,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 481 16 2525 2509 20.8609 0 - 3.00542 2014-07-01 16:24:08,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 482 16 2525 2509 20.8176 0 - 3.00542 2014-07-01 16:24:09,354.354 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 483 16 2525 2509 20.7745 0 - 3.00542 2014-07-01 16:24:10,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 484 16 2525 2509 20.7316 0 - 3.00542 2014-07-01 16:24:11,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 485 16 2525 2509 20.6888 0 - 3.00542 2014-07-01 16:24:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: finishing write tid 1 to mira0435601-140 2014-07-01 16:24:11,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: finishing write tid 2 to mira0435601-140 2014-07-01 16:24:11,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: finishing write tid 3 to mira0435601-140 2014-07-01 16:24:11,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: finishing write tid 5 to mira0435601-140 2014-07-01 16:24:11,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: finishing write tid 6 to mira0435601-140 2014-07-01 16:24:11,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 75 (ObjNum 756 snap 0 seq_num 756) dirty exists 2014-07-01 16:24:11,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: finishing write tid 1 to mira0435601-355 2014-07-01 16:24:11,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: finishing write tid 1 to mira0435601-116 2014-07-01 16:24:11,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: finishing write tid 2 to mira0435601-355 2014-07-01 16:24:11,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: finishing write tid 3 to mira0435601-116 2014-07-01 16:24:11,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: finishing write tid 4 to mira0435601-355 2014-07-01 16:24:12,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 486 16 2533 2517 20.7121 2.66667 0.603215 3.03822 2014-07-01 16:24:12,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: finishing write tid 4 to mira0435601-116 2014-07-01 16:24:12,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 81 (ObjNum 753 snap 0 seq_num 753) dirty exists 2014-07-01 16:24:13,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 1 to mira0435601-489 2014-07-01 16:24:13,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 2 to mira0435601-489 2014-07-01 16:24:13,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 3 to mira0435601-489 2014-07-01 16:24:13,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: expect (ObjNum 389 snap 0 seq_num 389) 2014-07-01 16:24:13,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 487 16 2551 2535 20.8174 72 0.665593 3.07186 2014-07-01 16:24:14,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 488 16 2551 2535 20.7747 0 - 3.07186 2014-07-01 16:24:15,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 489 16 2552 2536 20.7404 2 1.35835 3.07118 2014-07-01 16:24:15,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: finishing write tid 1 to mira0435601-338 2014-07-01 16:24:15,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: finishing write tid 2 to mira0435601-338 2014-07-01 16:24:16,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: finishing write tid 5 to mira0435601-355 2014-07-01 16:24:16,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 163 (ObjNum 754 snap 0 seq_num 754) dirty exists 2014-07-01 16:24:16,355.355 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 490 16 2556 2540 20.7307 16 3.13394 3.07167 2014-07-01 16:24:16,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: finishing write tid 3 to mira0435601-338 2014-07-01 16:24:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: finishing write tid 5 to mira0435601-338 2014-07-01 16:24:16,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: done (13 left) 2014-07-01 16:24:16,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: done (12 left) 2014-07-01 16:24:16,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: done (11 left) 2014-07-01 16:24:16,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: done (10 left) 2014-07-01 16:24:16,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:973: done (9 left) 2014-07-01 16:24:16,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:974: read oid 194 snap -1 2014-07-01 16:24:16,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 318 current snap is 0 2014-07-01 16:24:16,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: seq_num 764 ranges {1253376=610304,1863680=458752,2322432=733184,3055616=413696} 2014-07-01 16:24:17,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: writing mira0435601-318 from 1253376 to 1863680 tid 1 2014-07-01 16:24:17,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: writing mira0435601-318 from 1863680 to 2322432 tid 2 2014-07-01 16:24:17,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: writing mira0435601-318 from 2322432 to 3055616 tid 3 2014-07-01 16:24:17,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: writing mira0435601-318 from 3055616 to 3469312 tid 4 2014-07-01 16:24:17,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 420 current snap is 0 2014-07-01 16:24:17,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: seq_num 765 ranges {458752=479232,937984=794624,1732608=561152,2293760=249856} 2014-07-01 16:24:17,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: writing mira0435601-420 from 458752 to 937984 tid 1 2014-07-01 16:24:17,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: writing mira0435601-420 from 937984 to 1732608 tid 2 2014-07-01 16:24:17,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: writing mira0435601-420 from 1732608 to 2293760 tid 3 2014-07-01 16:24:17,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: writing mira0435601-420 from 2293760 to 2543616 tid 4 2014-07-01 16:24:17,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: read oid 179 snap -1 2014-07-01 16:24:17,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:978: read oid 399 snap -1 2014-07-01 16:24:17,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: expect (ObjNum 696 snap 0 seq_num 696) 2014-07-01 16:24:17,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 491 16 2564 2548 20.7536 32 3.97513 3.0735 2014-07-01 16:24:17,498.498 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:17,499.499 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:17,499.499 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 16:24:17,499.499 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:24:18,031.031 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:24:18,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:974: expect (ObjNum 504 snap 0 seq_num 504) 2014-07-01 16:24:18,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 492 16 2571 2555 20.7684 28 5.23539 3.07549 2014-07-01 16:24:19,339.339 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 26' 2014-07-01 16:24:19,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 493 16 2581 2565 20.8073 40 1.66026 3.07009 2014-07-01 16:24:20,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:978: expect (ObjNum 398 snap 0 seq_num 398) 2014-07-01 16:24:20,356.356 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 494 16 2586 2570 20.8057 20 0.212579 3.06601 2014-07-01 16:24:21,009.009 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 26 2014-07-01 16:24:21,357.357 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 495 16 2586 2570 20.7637 0 - 3.06601 2014-07-01 16:24:21,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 1 to mira0435601-420 2014-07-01 16:24:21,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 2 to mira0435601-420 2014-07-01 16:24:21,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 3 to mira0435601-420 2014-07-01 16:24:21,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 4 to mira0435601-420 2014-07-01 16:24:21,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 6 to mira0435601-420 2014-07-01 16:24:21,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 7 to mira0435601-420 2014-07-01 16:24:21,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 172 (ObjNum 765 snap 0 seq_num 765) dirty exists 2014-07-01 16:24:21,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: done (13 left) 2014-07-01 16:24:21,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:974: done (12 left) 2014-07-01 16:24:21,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: done (11 left) 2014-07-01 16:24:21,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:978: done (10 left) 2014-07-01 16:24:21,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: read oid 193 snap -1 2014-07-01 16:24:21,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 4 to mira0435601-489 2014-07-01 16:24:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 5 to mira0435601-489 2014-07-01 16:24:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 7 to mira0435601-489 2014-07-01 16:24:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: finishing write tid 1 to mira0435601-323 2014-07-01 16:24:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: expect (ObjNum 22 snap 0 seq_num 22) 2014-07-01 16:24:22,357.357 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 496 16 2586 2570 20.7218 0 - 3.06601 2014-07-01 16:24:22,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: finishing write tid 2 to mira0435601-323 2014-07-01 16:24:22,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: finishing write tid 4 to mira0435601-323 2014-07-01 16:24:22,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 1 to mira0435601-337 2014-07-01 16:24:22,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 2 to mira0435601-337 2014-07-01 16:24:22,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 3 to mira0435601-337 2014-07-01 16:24:22,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 4 to mira0435601-337 2014-07-01 16:24:22,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 6 to mira0435601-337 2014-07-01 16:24:22,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 1 to mira0435601-414 2014-07-01 16:24:22,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 1 to mira0435601-457 2014-07-01 16:24:22,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 3 to mira0435601-457 2014-07-01 16:24:22,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 2 to mira0435601-414 2014-07-01 16:24:22,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 3 to mira0435601-414 2014-07-01 16:24:22,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: done (10 left) 2014-07-01 16:24:22,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:980: read oid 373 snap -1 2014-07-01 16:24:22,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 4 to mira0435601-414 2014-07-01 16:24:22,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 6 to mira0435601-414 2014-07-01 16:24:22,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 7 to mira0435601-337 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 217 (ObjNum 757 snap 0 seq_num 757) dirty exists 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: finishing write tid 6 to mira0435601-338 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 167 (ObjNum 759 snap 0 seq_num 759) dirty exists 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: finishing write tid 7 to mira0435601-414 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 222 (ObjNum 758 snap 0 seq_num 758) dirty exists 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: finishing write tid 1 to mira0435601-427 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: finishing write tid 2 to mira0435601-427 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: finishing write tid 3 to mira0435601-427 2014-07-01 16:24:22,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: finishing write tid 5 to mira0435601-427 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: finishing write tid 6 to mira0435601-427 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 86 (ObjNum 760 snap 0 seq_num 760) dirty exists 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 4 to mira0435601-457 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 133 (ObjNum 761 snap 0 seq_num 761) dirty exists 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: finishing write tid 5 to mira0435601-323 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 139 (ObjNum 762 snap 0 seq_num 762) dirty exists 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 8 to mira0435601-489 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 136 (ObjNum 755 snap 0 seq_num 755) dirty exists 2014-07-01 16:24:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: expect (ObjNum 192 snap 0 seq_num 192) 2014-07-01 16:24:23,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: done (10 left) 2014-07-01 16:24:23,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: done (9 left) 2014-07-01 16:24:23,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: done (8 left) 2014-07-01 16:24:23,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: done (7 left) 2014-07-01 16:24:23,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: done (6 left) 2014-07-01 16:24:23,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: done (5 left) 2014-07-01 16:24:23,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: done (4 left) 2014-07-01 16:24:23,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: done (3 left) 2014-07-01 16:24:23,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:981: read oid 165 snap -1 2014-07-01 16:24:23,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: read oid 265 snap -1 2014-07-01 16:24:23,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:983: read oid 278 snap -1 2014-07-01 16:24:23,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 438 current snap is 0 2014-07-01 16:24:23,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: seq_num 766 ranges {1662976=507904,2170880=401408} 2014-07-01 16:24:23,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: writing mira0435601-438 from 1662976 to 2170880 tid 1 2014-07-01 16:24:23,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: writing mira0435601-438 from 2170880 to 2572288 tid 2 2014-07-01 16:24:23,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 247 current snap is 0 2014-07-01 16:24:23,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: seq_num 767 ranges {229376=430080,659456=610304,1269760=507904,1777664=86016} 2014-07-01 16:24:23,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: writing mira0435601-247 from 229376 to 659456 tid 1 2014-07-01 16:24:23,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: writing mira0435601-247 from 659456 to 1269760 tid 2 2014-07-01 16:24:23,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: writing mira0435601-247 from 1269760 to 1777664 tid 3 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: writing mira0435601-247 from 1777664 to 1863680 tid 4 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 45 current snap is 0 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: seq_num 768 ranges {1712128=700416,2412544=438272,2850816=364544} 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: writing mira0435601-45 from 1712128 to 2412544 tid 1 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: writing mira0435601-45 from 2412544 to 2850816 tid 2 2014-07-01 16:24:23,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: writing mira0435601-45 from 2850816 to 3215360 tid 3 2014-07-01 16:24:23,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 267 current snap is 0 2014-07-01 16:24:23,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: seq_num 769 ranges {1806336=327680} 2014-07-01 16:24:23,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: writing mira0435601-267 from 1806336 to 2134016 tid 1 2014-07-01 16:24:23,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: read oid 390 snap -1 2014-07-01 16:24:23,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:981: expect (ObjNum 164 snap 0 seq_num 164) 2014-07-01 16:24:23,413.413 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 497 16 2586 2570 20.6801 0 - 3.06601 2014-07-01 16:24:24,357.357 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 498 16 2600 2584 20.751 14 0.605672 3.07137 2014-07-01 16:24:24,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:983: expect (ObjNum 277 snap 0 seq_num 277) 2014-07-01 16:24:25,358.358 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 499 16 2602 2586 20.7255 8 1.03221 3.0698 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 1 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 2 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 3 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 4 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 6 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: finishing write tid 7 to mira0435601-247 2014-07-01 16:24:25,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 161 (ObjNum 767 snap 0 seq_num 767) dirty exists 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 1 to mira0435601-318 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 2 to mira0435601-318 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 3 to mira0435601-318 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:981: done (10 left) 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:983: done (9 left) 2014-07-01 16:24:25,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: done (8 left) 2014-07-01 16:24:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: read oid 67 snap -1 2014-07-01 16:24:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 4 to mira0435601-318 2014-07-01 16:24:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 98 current snap is 0 2014-07-01 16:24:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: seq_num 770 ranges {1060864=200704} 2014-07-01 16:24:25,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: writing mira0435601-98 from 1060864 to 1261568 tid 1 2014-07-01 16:24:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: read oid 428 snap -1 2014-07-01 16:24:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 6 to mira0435601-318 2014-07-01 16:24:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:980: expect (ObjNum 698 snap 0 seq_num 698) 2014-07-01 16:24:26,019.019 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:26,019.019 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:26,020.020 INFO:teuthology.task.thrashosds.thrasher:Adding osd 5 2014-07-01 16:24:26,020.020 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 5' 2014-07-01 16:24:26,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:24:26.270126min lat: 0.057269 max lat: 48.8206 avg lat: 3.06933 2014-07-01 16:24:26,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:24:26,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 500 16 2616 2600 20.796 56 2.2455 3.06933 2014-07-01 16:24:27,292.292 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.5. 2014-07-01 16:24:27,298.298 INFO:teuthology.task.thrashosds.thrasher:Added osd 5 2014-07-01 16:24:27,357.357 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 501 16 2618 2602 20.7705 8 1.08766 3.0675 2014-07-01 16:24:27,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 7 to mira0435601-318 2014-07-01 16:24:27,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 98 (ObjNum 764 snap 0 seq_num 764) dirty exists 2014-07-01 16:24:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: expect (ObjNum 389 snap 0 seq_num 389) 2014-07-01 16:24:27,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: expect (ObjNum 568 snap 0 seq_num 568) 2014-07-01 16:24:28,358.358 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 502 16 2622 2606 20.761 16 0.193897 3.06399 2014-07-01 16:24:29,358.358 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 503 16 2622 2606 20.7197 0 - 3.06399 2014-07-01 16:24:29,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: expect (ObjNum 66 snap 0 seq_num 66) 2014-07-01 16:24:30,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: finishing write tid 1 to mira0435601-98 2014-07-01 16:24:30,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: finishing write tid 3 to mira0435601-98 2014-07-01 16:24:30,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: finishing write tid 4 to mira0435601-98 2014-07-01 16:24:30,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 174 (ObjNum 770 snap 0 seq_num 770) dirty exists 2014-07-01 16:24:30,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: finishing write tid 1 to mira0435601-438 2014-07-01 16:24:30,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: finishing write tid 1 to mira0435601-267 2014-07-01 16:24:30,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: finishing write tid 3 to mira0435601-267 2014-07-01 16:24:30,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: finishing write tid 1 to mira0435601-45 2014-07-01 16:24:30,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: finishing write tid 2 to mira0435601-45 2014-07-01 16:24:30,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: finishing write tid 3 to mira0435601-45 2014-07-01 16:24:30,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: finishing write tid 5 to mira0435601-45 2014-07-01 16:24:30,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: finishing write tid 6 to mira0435601-45 2014-07-01 16:24:30,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 125 (ObjNum 768 snap 0 seq_num 768) dirty exists 2014-07-01 16:24:30,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: finishing write tid 2 to mira0435601-438 2014-07-01 16:24:30,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: finishing write tid 4 to mira0435601-438 2014-07-01 16:24:30,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: finishing write tid 4 to mira0435601-267 2014-07-01 16:24:30,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 141 (ObjNum 769 snap 0 seq_num 769) dirty exists 2014-07-01 16:24:30,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: expect (ObjNum 178 snap 0 seq_num 178) 2014-07-01 16:24:30,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 504 16 2622 2606 20.6786 0 - 3.06399 2014-07-01 16:24:31,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 505 16 2631 2615 20.7089 12 6.76494 3.06773 2014-07-01 16:24:31,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: finishing write tid 5 to mira0435601-438 2014-07-01 16:24:31,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 136 (ObjNum 766 snap 0 seq_num 766) dirty exists 2014-07-01 16:24:31,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: done (10 left) 2014-07-01 16:24:31,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: done (9 left) 2014-07-01 16:24:31,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:980: done (8 left) 2014-07-01 16:24:31,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: done (7 left) 2014-07-01 16:24:31,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: done (6 left) 2014-07-01 16:24:31,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: done (5 left) 2014-07-01 16:24:31,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: done (4 left) 2014-07-01 16:24:31,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: done (3 left) 2014-07-01 16:24:31,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: done (2 left) 2014-07-01 16:24:31,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: done (1 left) 2014-07-01 16:24:31,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 177 current snap is 0 2014-07-01 16:24:31,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: seq_num 771 ranges {2023424=520192,2543616=401408} 2014-07-01 16:24:31,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: writing mira0435601-177 from 2023424 to 2543616 tid 1 2014-07-01 16:24:31,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: writing mira0435601-177 from 2543616 to 2945024 tid 2 2014-07-01 16:24:31,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:993: read oid 407 snap -1 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: seq_num 772 ranges {1798144=548864,2347008=421888,2768896=782336,3551232=401408} 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: writing mira0435601-252 from 1798144 to 2347008 tid 1 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: writing mira0435601-252 from 2347008 to 2768896 tid 2 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: writing mira0435601-252 from 2768896 to 3551232 tid 3 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: writing mira0435601-252 from 3551232 to 3952640 tid 4 2014-07-01 16:24:31,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:995: delete oid 152 current snap is 0 2014-07-01 16:24:32,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 1 to mira0435601-252 2014-07-01 16:24:32,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 1 to mira0435601-177 2014-07-01 16:24:32,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 2 to mira0435601-252 2014-07-01 16:24:32,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 3 to mira0435601-252 2014-07-01 16:24:32,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 4 to mira0435601-252 2014-07-01 16:24:32,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 2 to mira0435601-177 2014-07-01 16:24:32,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 6 to mira0435601-252 2014-07-01 16:24:32,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 4 to mira0435601-177 2014-07-01 16:24:32,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 5 to mira0435601-177 2014-07-01 16:24:32,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 84 (ObjNum 771 snap 0 seq_num 771) dirty exists 2014-07-01 16:24:32,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 7 to mira0435601-252 2014-07-01 16:24:32,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 227 (ObjNum 772 snap 0 seq_num 772) dirty exists 2014-07-01 16:24:32,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: expect (ObjNum 427 snap 0 seq_num 427) 2014-07-01 16:24:32,298.298 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:32,298.298 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:32,298.298 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 16:24:32,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:24:32,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 506 16 2638 2622 20.7233 28 0.167794 3.06298 2014-07-01 16:24:32,874.874 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:24:33,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 507 16 2647 2631 20.7534 36 6.71116 3.0612 2014-07-01 16:24:34,255.255 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 16:24:34,358.358 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 508 16 2648 2632 20.7204 4 2.46325 3.06097 2014-07-01 16:24:34,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: done (4 left) 2014-07-01 16:24:34,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: done (3 left) 2014-07-01 16:24:34,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:993: done (2 left) 2014-07-01 16:24:34,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: done (1 left) 2014-07-01 16:24:34,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:995: done (0 left) 2014-07-01 16:24:34,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:996: delete oid 404 current snap is 0 2014-07-01 16:24:35,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 509 16 2651 2635 20.7033 12 1.88333 3.0626 2014-07-01 16:24:36,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 510 16 2661 2645 20.7411 40 1.79128 3.05776 2014-07-01 16:24:37,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 511 16 2661 2645 20.7005 0 - 3.05776 2014-07-01 16:24:38,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 512 16 2668 2652 20.7148 14 1.92558 3.05881 2014-07-01 16:24:39,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:996: done (0 left) 2014-07-01 16:24:39,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 317 current snap is 0 2014-07-01 16:24:39,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: seq_num 775 ranges {331776=720896} 2014-07-01 16:24:39,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: writing mira0435601-317 from 331776 to 1052672 tid 1 2014-07-01 16:24:39,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:998: read oid 371 snap -1 2014-07-01 16:24:39,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:999: delete oid 480 current snap is 0 2014-07-01 16:24:39,349.349 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 16:24:39,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 513 16 2677 2661 20.7446 36 4.7278 3.05578 2014-07-01 16:24:39,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:998: expect (ObjNum 576 snap 0 seq_num 576) 2014-07-01 16:24:40,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 514 16 2688 2672 20.7898 44 0.329495 3.07217 2014-07-01 16:24:41,362.362 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 515 16 2705 2689 20.8814 68 0.796627 3.06078 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: finishing write tid 1 to mira0435601-317 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: finishing write tid 3 to mira0435601-317 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: finishing write tid 4 to mira0435601-317 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 151 (ObjNum 775 snap 0 seq_num 775) dirty exists 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: done (2 left) 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:998: done (1 left) 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:999: done (0 left) 2014-07-01 16:24:41,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 442 current snap is 0 2014-07-01 16:24:41,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: seq_num 777 ranges {733184=512000,1245184=630784,1875968=172032} 2014-07-01 16:24:41,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: writing mira0435601-442 from 733184 to 1245184 tid 1 2014-07-01 16:24:41,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: writing mira0435601-442 from 1245184 to 1875968 tid 2 2014-07-01 16:24:41,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: writing mira0435601-442 from 1875968 to 2048000 tid 3 2014-07-01 16:24:41,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: read oid 235 snap -1 2014-07-01 16:24:41,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 155 current snap is 0 2014-07-01 16:24:41,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: seq_num 778 ranges {704512=139264} 2014-07-01 16:24:41,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: writing mira0435601-155 from 704512 to 843776 tid 1 2014-07-01 16:24:41,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 177 current snap is 0 2014-07-01 16:24:41,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: seq_num 779 ranges {2945024=565248,3510272=798720,4308992=4096} 2014-07-01 16:24:41,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: writing mira0435601-177 from 2945024 to 3510272 tid 1 2014-07-01 16:24:41,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: writing mira0435601-177 from 3510272 to 4308992 tid 2 2014-07-01 16:24:41,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: writing mira0435601-177 from 4308992 to 4313088 tid 3 2014-07-01 16:24:41,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: read oid 331 snap -1 2014-07-01 16:24:41,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: delete oid 381 current snap is 0 2014-07-01 16:24:41,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: expect (ObjNum 534 snap 0 seq_num 534) 2014-07-01 16:24:42,359.359 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 516 16 2705 2689 20.841 0 - 3.06078 2014-07-01 16:24:42,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: expect (ObjNum 330 snap 0 seq_num 330) 2014-07-01 16:24:43,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 517 16 2707 2691 20.8161 4 2.77393 3.06029 2014-07-01 16:24:43,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: done (5 left) 2014-07-01 16:24:43,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: done (4 left) 2014-07-01 16:24:43,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: done (3 left) 2014-07-01 16:24:43,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: read oid 474 snap -1 2014-07-01 16:24:43,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 122 current snap is 0 2014-07-01 16:24:43,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: seq_num 781 ranges {3145728=761856,3907584=610304,4517888=548864,5066752=45056} 2014-07-01 16:24:43,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: writing mira0435601-122 from 3145728 to 3907584 tid 1 2014-07-01 16:24:43,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: writing mira0435601-122 from 3907584 to 4517888 tid 2 2014-07-01 16:24:43,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: writing mira0435601-122 from 4517888 to 5066752 tid 3 2014-07-01 16:24:43,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: writing mira0435601-122 from 5066752 to 5111808 tid 4 2014-07-01 16:24:43,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1008: delete oid 124 current snap is 0 2014-07-01 16:24:43,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: expect (ObjNum 668 snap 0 seq_num 668) 2014-07-01 16:24:44,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 518 16 2707 2691 20.7759 0 - 3.06029 2014-07-01 16:24:44,362.362 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:44,362.362 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:44,362.362 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:24:44,362.362 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:24:44,958.958 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:24:45,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 519 16 2715 2699 20.7976 16 0.448353 3.06081 2014-07-01 16:24:45,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 1 to mira0435601-155 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 3 to mira0435601-155 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 4 to mira0435601-155 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 163 (ObjNum 778 snap 0 seq_num 778) dirty exists 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: finishing write tid 1 to mira0435601-177 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: finishing write tid 2 to mira0435601-177 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: finishing write tid 3 to mira0435601-177 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: finishing write tid 5 to mira0435601-177 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 1 to mira0435601-122 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 2 to mira0435601-122 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 3 to mira0435601-122 2014-07-01 16:24:45,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 4 to mira0435601-122 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 6 to mira0435601-122 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: finishing write tid 1 to mira0435601-442 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: finishing write tid 2 to mira0435601-442 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: finishing write tid 3 to mira0435601-442 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: finishing write tid 5 to mira0435601-442 2014-07-01 16:24:45,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: finishing write tid 6 to mira0435601-442 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 172 (ObjNum 777 snap 0 seq_num 777) dirty exists 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: finishing write tid 6 to mira0435601-177 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 88 (ObjNum 779 snap 0 seq_num 779) dirty exists 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: finishing write tid 7 to mira0435601-122 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 141 (ObjNum 781 snap 0 seq_num 781) dirty exists 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: done (5 left) 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: done (4 left) 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: done (3 left) 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: done (2 left) 2014-07-01 16:24:45,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: done (1 left) 2014-07-01 16:24:45,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1008: done (0 left) 2014-07-01 16:24:45,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:24:45,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: seq_num 783 ranges {1802240=503808,2306048=622592,2928640=446464,3375104=135168} 2014-07-01 16:24:45,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: writing mira0435601-284 from 1802240 to 2306048 tid 1 2014-07-01 16:24:45,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: writing mira0435601-284 from 2306048 to 2928640 tid 2 2014-07-01 16:24:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: writing mira0435601-284 from 2928640 to 3375104 tid 3 2014-07-01 16:24:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: writing mira0435601-284 from 3375104 to 3510272 tid 4 2014-07-01 16:24:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1010: read oid 347 snap -1 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: read oid 367 snap -1 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1012: read oid 345 snap -1 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 13 current snap is 0 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: seq_num 784 ranges {749568=532480} 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: writing mira0435601-13 from 749568 to 1282048 tid 1 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 475 current snap is 0 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: seq_num 785 ranges {2797568=204800} 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: writing mira0435601-475 from 2797568 to 3002368 tid 1 2014-07-01 16:24:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1015: read oid 78 snap -1 2014-07-01 16:24:45,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:1010: expect (ObjNum 346 snap 0 seq_num 346) 2014-07-01 16:24:46,328.328 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:24:46,328.328 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 66' 2014-07-01 16:24:46,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:24:46.273706min lat: 0.057269 max lat: 48.8206 avg lat: 3.06081 2014-07-01 16:24:46,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:24:46,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 520 16 2715 2699 20.7576 0 - 3.06081 2014-07-01 16:24:46,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: expect (ObjNum 366 snap 0 seq_num 366) 2014-07-01 16:24:47,360.360 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 521 16 2735 2719 20.8712 40 0.557447 3.0625 2014-07-01 16:24:47,608.608 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 66 2014-07-01 16:24:48,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 522 16 2745 2729 20.9079 40 0.550991 3.05499 2014-07-01 16:24:48,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:1015: expect (ObjNum 77 snap 0 seq_num 77) 2014-07-01 16:24:48,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:1012: expect (ObjNum 344 snap 0 seq_num 344) 2014-07-01 16:24:49,362.362 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 523 16 2747 2731 20.8832 8 2.59137 3.05444 2014-07-01 16:24:49,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: finishing write tid 1 to mira0435601-475 2014-07-01 16:24:49,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 1 to mira0435601-284 2014-07-01 16:24:49,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: finishing write tid 3 to mira0435601-475 2014-07-01 16:24:49,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 2 to mira0435601-284 2014-07-01 16:24:49,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 3 to mira0435601-284 2014-07-01 16:24:49,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 4 to mira0435601-284 2014-07-01 16:24:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 1 to mira0435601-13 2014-07-01 16:24:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 6 to mira0435601-284 2014-07-01 16:24:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 3 to mira0435601-13 2014-07-01 16:24:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1010: done (6 left) 2014-07-01 16:24:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: done (5 left) 2014-07-01 16:24:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1012: done (4 left) 2014-07-01 16:24:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1015: done (3 left) 2014-07-01 16:24:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: read oid 267 snap -1 2014-07-01 16:24:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: finishing write tid 4 to mira0435601-475 2014-07-01 16:24:49,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 165 (ObjNum 785 snap 0 seq_num 785) dirty exists 2014-07-01 16:24:49,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: finishing write tid 7 to mira0435601-284 2014-07-01 16:24:49,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 179 (ObjNum 783 snap 0 seq_num 783) dirty exists 2014-07-01 16:24:49,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 4 to mira0435601-13 2014-07-01 16:24:49,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 129 (ObjNum 784 snap 0 seq_num 784) dirty exists 2014-07-01 16:24:49,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: done (3 left) 2014-07-01 16:24:49,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: done (2 left) 2014-07-01 16:24:49,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: done (1 left) 2014-07-01 16:24:49,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: read oid 411 snap -1 2014-07-01 16:24:49,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1018: read oid 251 snap -1 2014-07-01 16:24:49,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 453 current snap is 0 2014-07-01 16:24:49,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: seq_num 786 ranges {1875968=77824} 2014-07-01 16:24:50,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: writing mira0435601-453 from 1875968 to 1953792 tid 1 2014-07-01 16:24:50,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: read oid 219 snap -1 2014-07-01 16:24:50,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: seq_num 787 ranges {2908160=753664,3661824=602112,4263936=495616,4759552=507904} 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: writing mira0435601-391 from 2908160 to 3661824 tid 1 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: writing mira0435601-391 from 3661824 to 4263936 tid 2 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: writing mira0435601-391 from 4263936 to 4759552 tid 3 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: writing mira0435601-391 from 4759552 to 5267456 tid 4 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: read oid 332 snap -1 2014-07-01 16:24:50,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: expect (ObjNum 769 snap 0 seq_num 769) 2014-07-01 16:24:50,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 524 16 2747 2731 20.8433 0 - 3.05444 2014-07-01 16:24:51,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: expect (ObjNum 218 snap 0 seq_num 218) 2014-07-01 16:24:51,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 525 16 2749 2733 20.8189 4 2.25507 3.05406 2014-07-01 16:24:52,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: expect (ObjNum 601 snap 0 seq_num 601) 2014-07-01 16:24:52,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 526 16 2753 2737 20.8097 16 0.140215 3.05482 2014-07-01 16:24:52,620.620 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:52,620.620 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:52,620.620 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.348315 2014-07-01 16:24:52,621.621 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 0.348314970602' 2014-07-01 16:24:53,362.362 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 527 16 2765 2749 20.8613 48 0.117075 3.05299 2014-07-01 16:24:53,587.587 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 0.348315 (8228272) 2014-07-01 16:24:53,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: expect (ObjNum 331 snap 0 seq_num 331) 2014-07-01 16:24:54,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1018: expect (ObjNum 250 snap 0 seq_num 250) 2014-07-01 16:24:54,361.361 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 528 16 2772 2756 20.8748 28 0.065487 3.04884 2014-07-01 16:24:55,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: finishing write tid 1 to mira0435601-453 2014-07-01 16:24:55,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: finishing write tid 3 to mira0435601-453 2014-07-01 16:24:55,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 1 to mira0435601-391 2014-07-01 16:24:55,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 2 to mira0435601-391 2014-07-01 16:24:55,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 3 to mira0435601-391 2014-07-01 16:24:55,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 4 to mira0435601-391 2014-07-01 16:24:55,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 6 to mira0435601-391 2014-07-01 16:24:55,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: done (6 left) 2014-07-01 16:24:55,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: done (5 left) 2014-07-01 16:24:55,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1018: done (4 left) 2014-07-01 16:24:55,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: done (3 left) 2014-07-01 16:24:55,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: done (2 left) 2014-07-01 16:24:55,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: read oid 288 snap -1 2014-07-01 16:24:55,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 258 current snap is 0 2014-07-01 16:24:55,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: seq_num 788 ranges {1994752=778240,2772992=401408} 2014-07-01 16:24:55,362.362 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 529 16 2772 2756 20.8353 0 - 3.04884 2014-07-01 16:24:55,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: writing mira0435601-258 from 1994752 to 2772992 tid 1 2014-07-01 16:24:55,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: writing mira0435601-258 from 2772992 to 3174400 tid 2 2014-07-01 16:24:55,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1025: delete oid 14 current snap is 0 2014-07-01 16:24:56,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: finishing write tid 1 to mira0435601-258 2014-07-01 16:24:56,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: finishing write tid 2 to mira0435601-258 2014-07-01 16:24:56,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: finishing write tid 4 to mira0435601-258 2014-07-01 16:24:56,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: done (4 left) 2014-07-01 16:24:56,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1025: done (3 left) 2014-07-01 16:24:56,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 131 current snap is 0 2014-07-01 16:24:56,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: seq_num 790 ranges {0=573440,573440=598016,1171456=516096,1687552=90112} 2014-07-01 16:24:56,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: writing mira0435601-131 from 0 to 573440 tid 1 2014-07-01 16:24:56,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: writing mira0435601-131 from 573440 to 1171456 tid 2 2014-07-01 16:24:56,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: writing mira0435601-131 from 1171456 to 1687552 tid 3 2014-07-01 16:24:56,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: writing mira0435601-131 from 1687552 to 1777664 tid 4 2014-07-01 16:24:56,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:1027: read oid 292 snap -1 2014-07-01 16:24:56,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: read oid 78 snap -1 2014-07-01 16:24:56,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 16:24:56,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: seq_num 791 ranges {28672=598016} 2014-07-01 16:24:56,459.459 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 530 16 2779 2763 20.8489 14 2.38384 3.04924 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: writing mira0435601-351 from 28672 to 626688 tid 1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 60 current snap is 0 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: seq_num 792 ranges {1761280=471040} 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: writing mira0435601-60 from 1761280 to 2232320 tid 1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: read oid 483 snap -1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: seq_num 793 ranges {0=344064} 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: writing mira0435601-288 from 0 to 344064 tid 1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: read oid 214 snap -1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 386 current snap is 0 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: seq_num 794 ranges {3096576=16384} 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: writing mira0435601-386 from 3096576 to 3112960 tid 1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1035: read oid 374 snap -1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1036: read oid 393 snap -1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1037: read oid 3 snap -1 2014-07-01 16:24:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 318 current snap is 0 2014-07-01 16:24:56,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: seq_num 795 ranges {3469312=692224,4161536=409600,4571136=634880,5206016=561152} 2014-07-01 16:24:56,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: writing mira0435601-318 from 3469312 to 4161536 tid 1 2014-07-01 16:24:56,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: writing mira0435601-318 from 4161536 to 4571136 tid 2 2014-07-01 16:24:56,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: writing mira0435601-318 from 4571136 to 5206016 tid 3 2014-07-01 16:24:56,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: writing mira0435601-318 from 5206016 to 5767168 tid 4 2014-07-01 16:24:56,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:24:56,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1035: expect (ObjNum 373 snap 0 seq_num 373) 2014-07-01 16:24:56,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: finishing write tid 1 to mira0435601-351 2014-07-01 16:24:56,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: finishing write tid 3 to mira0435601-351 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: finishing write tid 1 to mira0435601-288 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: finishing write tid 1 to mira0435601-60 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: finishing write tid 4 to mira0435601-453 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 158 (ObjNum 786 snap 0 seq_num 786) dirty exists 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: finishing write tid 3 to mira0435601-60 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: finishing write tid 3 to mira0435601-288 2014-07-01 16:24:56,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1027: expect (ObjNum 664 snap 0 seq_num 664) 2014-07-01 16:24:57,363.363 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 531 16 2797 2781 20.9452 72 5.83596 3.05356 2014-07-01 16:24:58,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 532 16 2805 2789 20.9659 32 0.60202 3.04674 2014-07-01 16:24:58,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: finishing write tid 7 to mira0435601-391 2014-07-01 16:24:58,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 163 (ObjNum 787 snap 0 seq_num 787) dirty exists 2014-07-01 16:24:58,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: expect (ObjNum 77 snap 0 seq_num 77) 2014-07-01 16:24:58,596.596 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:24:58,596.596 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:24:58,596.596 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:24:58,596.596 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:24:58,596.596 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:24:58,596.596 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:24:58,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 1 to mira0435601-131 2014-07-01 16:24:58,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 2 to mira0435601-131 2014-07-01 16:24:58,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 3 to mira0435601-131 2014-07-01 16:24:58,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 4 to mira0435601-131 2014-07-01 16:24:58,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: expect (ObjNum 213 snap 0 seq_num 213) 2014-07-01 16:24:59,363.363 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 533 16 2809 2793 20.9566 16 1.44579 3.04427 2014-07-01 16:24:59,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 6 to mira0435601-131 2014-07-01 16:24:59,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: finishing write tid 1 to mira0435601-386 2014-07-01 16:24:59,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: finishing write tid 3 to mira0435601-386 2014-07-01 16:24:59,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1037: expect (ObjNum 537 snap 0 seq_num 537) 2014-07-01 16:25:00,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: finishing write tid 5 to mira0435601-258 2014-07-01 16:25:00,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 258 v 175 (ObjNum 788 snap 0 seq_num 788) dirty exists 2014-07-01 16:25:00,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: finishing write tid 7 to mira0435601-131 2014-07-01 16:25:00,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 180 (ObjNum 790 snap 0 seq_num 790) dirty exists 2014-07-01 16:25:00,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: finishing write tid 4 to mira0435601-386 2014-07-01 16:25:00,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 181 (ObjNum 794 snap 0 seq_num 794) dirty exists 2014-07-01 16:25:00,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: expect (ObjNum 482 snap 0 seq_num 482) 2014-07-01 16:25:00,362.362 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 534 16 2811 2795 20.9323 8 0.889691 3.04315 2014-07-01 16:25:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: finishing write tid 4 to mira0435601-351 2014-07-01 16:25:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 229 (ObjNum 791 snap 0 seq_num 791) dirty exists 2014-07-01 16:25:01,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: finishing write tid 4 to mira0435601-288 2014-07-01 16:25:01,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 231 (ObjNum 793 snap 0 seq_num 793) dirty exists 2014-07-01 16:25:01,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1036: expect (ObjNum 392 snap 0 seq_num 392) 2014-07-01 16:25:01,363.363 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 535 16 2812 2796 20.9007 4 1.6943 3.04267 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: finishing write tid 4 to mira0435601-60 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 60 v 90 (ObjNum 792 snap 0 seq_num 792) dirty exists 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: done (15 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: done (14 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: done (13 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: done (12 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1027: done (11 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: done (10 left) 2014-07-01 16:25:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: done (9 left) 2014-07-01 16:25:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: done (8 left) 2014-07-01 16:25:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: done (7 left) 2014-07-01 16:25:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: done (6 left) 2014-07-01 16:25:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: done (5 left) 2014-07-01 16:25:01,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: done (4 left) 2014-07-01 16:25:01,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1035: done (3 left) 2014-07-01 16:25:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1036: done (2 left) 2014-07-01 16:25:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1037: done (1 left) 2014-07-01 16:25:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 446 current snap is 0 2014-07-01 16:25:01,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: seq_num 796 ranges {1822720=565248,2387968=638976,3026944=507904,3534848=458752} 2014-07-01 16:25:01,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0435601-446 from 1822720 to 2387968 tid 1 2014-07-01 16:25:01,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0435601-446 from 2387968 to 3026944 tid 2 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0435601-446 from 3026944 to 3534848 tid 3 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0435601-446 from 3534848 to 3993600 tid 4 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: seq_num 797 ranges {3563520=589824,4153344=401408} 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: writing mira0435601-147 from 3563520 to 4153344 tid 1 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: writing mira0435601-147 from 4153344 to 4554752 tid 2 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: read oid 218 snap -1 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 343 current snap is 0 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: seq_num 798 ranges {2510848=512000,3022848=405504,3428352=405504,3833856=393216} 2014-07-01 16:25:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: writing mira0435601-343 from 2510848 to 3022848 tid 1 2014-07-01 16:25:01,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: writing mira0435601-343 from 3022848 to 3428352 tid 2 2014-07-01 16:25:01,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: writing mira0435601-343 from 3428352 to 3833856 tid 3 2014-07-01 16:25:01,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: writing mira0435601-343 from 3833856 to 4227072 tid 4 2014-07-01 16:25:01,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1043: read oid 189 snap -1 2014-07-01 16:25:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 274 current snap is 0 2014-07-01 16:25:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: seq_num 799 ranges {2068480=540672} 2014-07-01 16:25:01,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: writing mira0435601-274 from 2068480 to 2609152 tid 1 2014-07-01 16:25:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: read oid 479 snap -1 2014-07-01 16:25:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1046: read oid 94 snap -1 2014-07-01 16:25:01,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: expect (ObjNum 217 snap 0 seq_num 217) 2014-07-01 16:25:02,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1043: expect (ObjNum 188 snap 0 seq_num 188) 2014-07-01 16:25:02,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 536 16 2812 2796 20.8617 0 - 3.04267 2014-07-01 16:25:03,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: expect (ObjNum 478 snap 0 seq_num 478) 2014-07-01 16:25:03,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 537 15 2820 2805 20.8899 18 4.1115 3.0511 2014-07-01 16:25:04,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 538 16 2829 2813 20.9105 32 0.212165 3.054 2014-07-01 16:25:04,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:1046: expect (ObjNum 615 snap 0 seq_num 615) 2014-07-01 16:25:05,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 539 16 2851 2835 21.035 88 0.57943 3.03974 2014-07-01 16:25:06,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:25:06.276376min lat: 0.057269 max lat: 48.8206 avg lat: 3.03208 2014-07-01 16:25:06,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:25:06,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 540 16 2863 2847 21.0849 48 0.250682 3.03208 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 1 to mira0435601-446 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 2 to mira0435601-446 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 3 to mira0435601-446 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 4 to mira0435601-446 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 6 to mira0435601-446 2014-07-01 16:25:07,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 7 to mira0435601-446 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 185 (ObjNum 796 snap 0 seq_num 796) dirty exists 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: finishing write tid 1 to mira0435601-147 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: finishing write tid 2 to mira0435601-147 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: finishing write tid 4 to mira0435601-147 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 1 to mira0435601-274 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 3 to mira0435601-274 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: finishing write tid 5 to mira0435601-147 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 95 (ObjNum 797 snap 0 seq_num 797) dirty exists 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 4 to mira0435601-274 2014-07-01 16:25:07,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 118 (ObjNum 799 snap 0 seq_num 799) dirty exists 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: done (8 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: done (7 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: done (6 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1043: done (5 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: done (4 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: done (3 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1046: done (2 left) 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1047: read oid 433 snap -1 2014-07-01 16:25:07,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 1 to mira0435601-343 2014-07-01 16:25:07,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 2 to mira0435601-343 2014-07-01 16:25:07,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 3 to mira0435601-343 2014-07-01 16:25:07,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 4 to mira0435601-343 2014-07-01 16:25:07,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 6 to mira0435601-343 2014-07-01 16:25:07,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: finishing write tid 7 to mira0435601-343 2014-07-01 16:25:07,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 168 (ObjNum 798 snap 0 seq_num 798) dirty exists 2014-07-01 16:25:07,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: done (2 left) 2014-07-01 16:25:07,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 340 current snap is 0 2014-07-01 16:25:07,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: seq_num 800 ranges {0=655360,655360=622592,1277952=180224} 2014-07-01 16:25:07,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: writing mira0435601-340 from 0 to 655360 tid 1 2014-07-01 16:25:07,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: writing mira0435601-340 from 655360 to 1277952 tid 2 2014-07-01 16:25:07,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: writing mira0435601-340 from 1277952 to 1458176 tid 3 2014-07-01 16:25:07,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 283 current snap is 0 2014-07-01 16:25:07,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: seq_num 801 ranges {0=282624} 2014-07-01 16:25:07,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: writing mira0435601-283 from 0 to 282624 tid 1 2014-07-01 16:25:07,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 16:25:07,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: seq_num 802 ranges {278528=708608,987136=786432,1773568=16384} 2014-07-01 16:25:07,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: writing mira0435601-444 from 278528 to 987136 tid 1 2014-07-01 16:25:07,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: writing mira0435601-444 from 987136 to 1773568 tid 2 2014-07-01 16:25:07,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: writing mira0435601-444 from 1773568 to 1789952 tid 3 2014-07-01 16:25:07,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1051: read oid 51 snap -1 2014-07-01 16:25:07,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1052: read oid 447 snap -1 2014-07-01 16:25:07,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:25:07,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: seq_num 803 ranges {0=475136,475136=401408} 2014-07-01 16:25:07,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: writing mira0435601-21 from 0 to 475136 tid 1 2014-07-01 16:25:07,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: writing mira0435601-21 from 475136 to 876544 tid 2 2014-07-01 16:25:07,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: read oid 356 snap -1 2014-07-01 16:25:07,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1055: read oid 407 snap -1 2014-07-01 16:25:07,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1051: expect (ObjNum 50 snap 0 seq_num 50) 2014-07-01 16:25:07,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 541 16 2866 2850 21.0681 12 2.08424 3.03047 2014-07-01 16:25:08,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: expect (ObjNum 355 snap 0 seq_num 355) 2014-07-01 16:25:08,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 542 16 2871 2855 21.0661 20 1.21368 3.02835 2014-07-01 16:25:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:1051: done (9 left) 2014-07-01 16:25:08,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: done (8 left) 2014-07-01 16:25:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:1055: done (7 left) 2014-07-01 16:25:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 253 current snap is 0 2014-07-01 16:25:08,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: seq_num 804 ranges {1671168=503808,2174976=602112,2777088=802816,3579904=200704} 2014-07-01 16:25:08,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0435601-253 from 1671168 to 2174976 tid 1 2014-07-01 16:25:08,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0435601-253 from 2174976 to 2777088 tid 2 2014-07-01 16:25:08,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0435601-253 from 2777088 to 3579904 tid 3 2014-07-01 16:25:08,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0435601-253 from 3579904 to 3780608 tid 4 2014-07-01 16:25:08,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 16:25:08,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: seq_num 805 ranges {1748992=528384,2277376=401408} 2014-07-01 16:25:08,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: writing mira0435601-394 from 1748992 to 2277376 tid 1 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: writing mira0435601-394 from 2277376 to 2678784 tid 2 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1058: read oid 204 snap -1 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1059: read oid 404 snap -1 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1060: read oid 359 snap -1 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 298 current snap is 0 2014-07-01 16:25:08,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: seq_num 806 ranges {2084864=647168,2732032=614400,3346432=503808,3850240=86016} 2014-07-01 16:25:08,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: writing mira0435601-298 from 2084864 to 2732032 tid 1 2014-07-01 16:25:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: writing mira0435601-298 from 2732032 to 3346432 tid 2 2014-07-01 16:25:08,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: writing mira0435601-298 from 3346432 to 3850240 tid 3 2014-07-01 16:25:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: writing mira0435601-298 from 3850240 to 3936256 tid 4 2014-07-01 16:25:08,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:1062: delete oid 401 current snap is 0 2014-07-01 16:25:08,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:1058: expect (ObjNum 722 snap 0 seq_num 722) 2014-07-01 16:25:09,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 543 16 2874 2858 21.0494 12 2.23198 3.02768 2014-07-01 16:25:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: finishing write tid 1 to mira0435601-21 2014-07-01 16:25:09,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: finishing write tid 2 to mira0435601-21 2014-07-01 16:25:09,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: finishing write tid 4 to mira0435601-21 2014-07-01 16:25:09,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: finishing write tid 5 to mira0435601-21 2014-07-01 16:25:09,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 123 (ObjNum 803 snap 0 seq_num 803) dirty exists 2014-07-01 16:25:10,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 1 to mira0435601-318 2014-07-01 16:25:10,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:1047: expect (ObjNum 432 snap 0 seq_num 432) 2014-07-01 16:25:10,365.365 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 544 16 2886 2870 21.099 48 0.163216 3.02751 2014-07-01 16:25:10,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 2 to mira0435601-318 2014-07-01 16:25:10,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 3 to mira0435601-318 2014-07-01 16:25:10,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 4 to mira0435601-318 2014-07-01 16:25:10,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 6 to mira0435601-318 2014-07-01 16:25:10,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1052: expect (ObjNum 446 snap 0 seq_num 446) 2014-07-01 16:25:11,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: finishing write tid 1 to mira0435601-340 2014-07-01 16:25:11,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: finishing write tid 2 to mira0435601-340 2014-07-01 16:25:11,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: finishing write tid 3 to mira0435601-340 2014-07-01 16:25:11,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: finishing write tid 5 to mira0435601-340 2014-07-01 16:25:11,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: finishing write tid 6 to mira0435601-340 2014-07-01 16:25:11,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 129 (ObjNum 800 snap 0 seq_num 800) dirty exists 2014-07-01 16:25:11,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:1060: expect (ObjNum 358 snap 0 seq_num 358) 2014-07-01 16:25:11,363.363 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 545 16 2889 2873 21.0823 12 3.55159 3.02842 2014-07-01 16:25:12,365.365 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 546 16 2889 2873 21.0436 0 - 3.02842 2014-07-01 16:25:13,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 1 to mira0435601-283 2014-07-01 16:25:13,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 3 to mira0435601-283 2014-07-01 16:25:13,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 1 to mira0435601-253 2014-07-01 16:25:13,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 2 to mira0435601-253 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 3 to mira0435601-253 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 4 to mira0435601-253 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 6 to mira0435601-253 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: finishing write tid 1 to mira0435601-394 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: finishing write tid 1 to mira0435601-444 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1047: done (13 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: done (12 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1052: done (11 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: done (10 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1058: done (9 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1059: done (8 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1060: done (7 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1062: done (6 left) 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1063: delete oid 239 current snap is 0 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: finishing write tid 2 to mira0435601-444 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: finishing write tid 3 to mira0435601-444 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: finishing write tid 5 to mira0435601-444 2014-07-01 16:25:13,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: finishing write tid 2 to mira0435601-394 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: finishing write tid 4 to mira0435601-394 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 1 to mira0435601-298 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 2 to mira0435601-298 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 3 to mira0435601-298 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 4 to mira0435601-298 2014-07-01 16:25:13,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 6 to mira0435601-298 2014-07-01 16:25:13,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1063: done (6 left) 2014-07-01 16:25:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1064: read oid 172 snap -1 2014-07-01 16:25:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 16:25:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: seq_num 809 ranges {1187840=466944,1654784=409600,2064384=393216} 2014-07-01 16:25:13,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: writing mira0435601-97 from 1187840 to 1654784 tid 1 2014-07-01 16:25:13,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: writing mira0435601-97 from 1654784 to 2064384 tid 2 2014-07-01 16:25:13,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: writing mira0435601-97 from 2064384 to 2457600 tid 3 2014-07-01 16:25:13,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: read oid 378 snap -1 2014-07-01 16:25:13,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1064: expect (ObjNum 673 snap 0 seq_num 673) 2014-07-01 16:25:13,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 547 16 2909 2893 21.1514 40 0.40468 3.02346 2014-07-01 16:25:15,240.240 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 548 16 2917 2901 21.1712 32 0.70876 3.01688 2014-07-01 16:25:15,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 549 16 2926 2910 21.1982 36 1.8047 3.01295 2014-07-01 16:25:15,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: expect (ObjNum 377 snap 0 seq_num 377) 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: finishing write tid 7 to mira0435601-318 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 104 (ObjNum 795 snap 0 seq_num 795) dirty exists 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: finishing write tid 1 to mira0435601-97 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: finishing write tid 2 to mira0435601-97 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: finishing write tid 3 to mira0435601-97 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: finishing write tid 5 to mira0435601-97 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 4 to mira0435601-283 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 233 (ObjNum 801 snap 0 seq_num 801) dirty exists 2014-07-01 16:25:15,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 7 to mira0435601-253 2014-07-01 16:25:15,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 238 (ObjNum 804 snap 0 seq_num 804) dirty exists 2014-07-01 16:25:15,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: finishing write tid 5 to mira0435601-394 2014-07-01 16:25:15,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 143 (ObjNum 805 snap 0 seq_num 805) dirty exists 2014-07-01 16:25:15,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: finishing write tid 6 to mira0435601-444 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 133 (ObjNum 802 snap 0 seq_num 802) dirty exists 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: finishing write tid 7 to mira0435601-298 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 80 (ObjNum 806 snap 0 seq_num 806) dirty exists 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: done (8 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: done (7 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: done (6 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: done (5 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: done (4 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: done (3 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1064: done (2 left) 2014-07-01 16:25:15,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: done (1 left) 2014-07-01 16:25:15,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 261 current snap is 0 2014-07-01 16:25:15,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: seq_num 810 ranges {561152=94208} 2014-07-01 16:25:15,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: writing mira0435601-261 from 561152 to 655360 tid 1 2014-07-01 16:25:15,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 16:25:15,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: seq_num 811 ranges {1064960=520192,1585152=577536,2162688=225280} 2014-07-01 16:25:15,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: writing mira0435601-327 from 1064960 to 1585152 tid 1 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: writing mira0435601-327 from 1585152 to 2162688 tid 2 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: writing mira0435601-327 from 2162688 to 2387968 tid 3 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 436 current snap is 0 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: seq_num 812 ranges {339968=442368,782336=405504,1187840=450560,1638400=434176,2072576=315392} 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: writing mira0435601-436 from 339968 to 782336 tid 1 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: writing mira0435601-436 from 782336 to 1187840 tid 2 2014-07-01 16:25:16,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: writing mira0435601-436 from 1187840 to 1638400 tid 3 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: writing mira0435601-436 from 1638400 to 2072576 tid 4 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: writing mira0435601-436 from 2072576 to 2387968 tid 5 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: read oid 365 snap -1 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: seq_num 813 ranges {135168=716800,851968=794624,1646592=409600} 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: writing mira0435601-311 from 135168 to 851968 tid 1 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: writing mira0435601-311 from 851968 to 1646592 tid 2 2014-07-01 16:25:16,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: writing mira0435601-311 from 1646592 to 2056192 tid 3 2014-07-01 16:25:16,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 160 current snap is 0 2014-07-01 16:25:16,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: seq_num 814 ranges {2330624=741376} 2014-07-01 16:25:16,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: writing mira0435601-160 from 2330624 to 3072000 tid 1 2014-07-01 16:25:16,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 131 current snap is 0 2014-07-01 16:25:16,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: seq_num 815 ranges {1777664=614400} 2014-07-01 16:25:16,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: writing mira0435601-131 from 1777664 to 2392064 tid 1 2014-07-01 16:25:16,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 331 current snap is 0 2014-07-01 16:25:16,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: seq_num 816 ranges {1712128=536576,2248704=634880,2883584=167936} 2014-07-01 16:25:16,073.073 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:25:16,073.073 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:25:16,074.074 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 16:25:16,074.074 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:25:16,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: writing mira0435601-331 from 1712128 to 2248704 tid 1 2014-07-01 16:25:16,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: writing mira0435601-331 from 2248704 to 2883584 tid 2 2014-07-01 16:25:16,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: writing mira0435601-331 from 2883584 to 3051520 tid 3 2014-07-01 16:25:16,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1075: read oid 479 snap -1 2014-07-01 16:25:16,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: expect (ObjNum 364 snap 0 seq_num 364) 2014-07-01 16:25:16,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1075: expect (ObjNum 478 snap 0 seq_num 478) 2014-07-01 16:25:16,364.364 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 550 16 2929 2913 21.1815 12 1.17927 3.01097 2014-07-01 16:25:16,756.756 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:25:17,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 551 16 2935 2919 21.1866 24 1.33608 3.00738 2014-07-01 16:25:17,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: finishing write tid 6 to mira0435601-97 2014-07-01 16:25:17,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 147 (ObjNum 809 snap 0 seq_num 809) dirty exists 2014-07-01 16:25:17,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: finishing write tid 1 to mira0435601-261 2014-07-01 16:25:17,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: done (9 left) 2014-07-01 16:25:17,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: done (8 left) 2014-07-01 16:25:17,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1075: done (7 left) 2014-07-01 16:25:17,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1076: read oid 16 snap -1 2014-07-01 16:25:17,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: finishing write tid 3 to mira0435601-261 2014-07-01 16:25:17,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: finishing write tid 4 to mira0435601-261 2014-07-01 16:25:17,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 149 (ObjNum 810 snap 0 seq_num 810) dirty exists 2014-07-01 16:25:17,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: done (7 left) 2014-07-01 16:25:17,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 13 current snap is 0 2014-07-01 16:25:17,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: seq_num 817 ranges {1282048=159744} 2014-07-01 16:25:17,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: writing mira0435601-13 from 1282048 to 1441792 tid 1 2014-07-01 16:25:17,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1078: read oid 438 snap -1 2014-07-01 16:25:17,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1076: expect (ObjNum 15 snap 0 seq_num 15) 2014-07-01 16:25:17,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1078: expect (ObjNum 766 snap 0 seq_num 766) 2014-07-01 16:25:17,949.949 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 44' 2014-07-01 16:25:18,365.365 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 552 16 2938 2922 21.1699 12 1.7658 3.00646 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: finishing write tid 1 to mira0435601-327 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: finishing write tid 2 to mira0435601-327 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: finishing write tid 3 to mira0435601-327 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: finishing write tid 5 to mira0435601-327 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 1 to mira0435601-436 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 2 to mira0435601-436 2014-07-01 16:25:19,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1076: done (8 left) 2014-07-01 16:25:19,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1078: done (7 left) 2014-07-01 16:25:19,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:25:19,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: seq_num 818 ranges {3526656=32768} 2014-07-01 16:25:19,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: writing mira0435601-172 from 3526656 to 3559424 tid 1 2014-07-01 16:25:19,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 412 current snap is 0 2014-07-01 16:25:19,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: seq_num 819 ranges {1372160=708608,2080768=786432,2867200=516096,3383296=303104} 2014-07-01 16:25:19,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: writing mira0435601-412 from 1372160 to 2080768 tid 1 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: writing mira0435601-412 from 2080768 to 2867200 tid 2 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: writing mira0435601-412 from 2867200 to 3383296 tid 3 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: writing mira0435601-412 from 3383296 to 3686400 tid 4 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: seq_num 820 ranges {876544=733184,1609728=401408} 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: writing mira0435601-21 from 876544 to 1609728 tid 1 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: writing mira0435601-21 from 1609728 to 2011136 tid 2 2014-07-01 16:25:19,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: read oid 434 snap -1 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 3 to mira0435601-436 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 4 to mira0435601-436 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 5 to mira0435601-436 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 7 to mira0435601-436 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: finishing write tid 1 to mira0435601-131 2014-07-01 16:25:19,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: finishing write tid 3 to mira0435601-131 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: finishing write tid 1 to mira0435601-331 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: finishing write tid 2 to mira0435601-331 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: finishing write tid 3 to mira0435601-331 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: finishing write tid 5 to mira0435601-331 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: finishing write tid 6 to mira0435601-327 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 127 (ObjNum 811 snap 0 seq_num 811) dirty exists 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: finishing write tid 1 to mira0435601-13 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: finishing write tid 3 to mira0435601-13 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: finishing write tid 8 to mira0435601-436 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 133 (ObjNum 812 snap 0 seq_num 812) dirty exists 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: finishing write tid 4 to mira0435601-131 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 187 (ObjNum 815 snap 0 seq_num 815) dirty exists 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: finishing write tid 4 to mira0435601-13 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 135 (ObjNum 817 snap 0 seq_num 817) dirty exists 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: finishing write tid 6 to mira0435601-331 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 191 (ObjNum 816 snap 0 seq_num 816) dirty exists 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: done (10 left) 2014-07-01 16:25:19,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: done (9 left) 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: done (8 left) 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: done (7 left) 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: done (6 left) 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1083: read oid 16 snap -1 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 16:25:19,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: seq_num 821 ranges {2678784=655360,3334144=598016,3932160=462848,4395008=143360} 2014-07-01 16:25:19,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: writing mira0435601-394 from 2678784 to 3334144 tid 1 2014-07-01 16:25:19,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: writing mira0435601-394 from 3334144 to 3932160 tid 2 2014-07-01 16:25:19,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 553 16 2947 2931 21.1967 36 5.24206 3.01101 2014-07-01 16:25:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: writing mira0435601-394 from 3932160 to 4395008 tid 3 2014-07-01 16:25:19,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: writing mira0435601-394 from 4395008 to 4538368 tid 4 2014-07-01 16:25:19,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:1085: delete oid 259 current snap is 0 2014-07-01 16:25:19,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: expect (ObjNum 580 snap 0 seq_num 580) 2014-07-01 16:25:19,835.835 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 44 2014-07-01 16:25:20,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1083: expect (ObjNum 15 snap 0 seq_num 15) 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: finishing write tid 1 to mira0435601-160 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: finishing write tid 3 to mira0435601-160 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: finishing write tid 1 to mira0435601-311 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: finishing write tid 1 to mira0435601-21 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: finishing write tid 2 to mira0435601-311 2014-07-01 16:25:20,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: finishing write tid 3 to mira0435601-311 2014-07-01 16:25:20,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: done (8 left) 2014-07-01 16:25:20,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1083: done (7 left) 2014-07-01 16:25:20,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1085: done (6 left) 2014-07-01 16:25:20,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1086: delete oid 393 current snap is 0 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: finishing write tid 2 to mira0435601-21 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: finishing write tid 4 to mira0435601-21 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: finishing write tid 5 to mira0435601-311 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: finishing write tid 5 to mira0435601-21 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 136 (ObjNum 820 snap 0 seq_num 820) dirty exists 2014-07-01 16:25:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: finishing write tid 1 to mira0435601-172 2014-07-01 16:25:20,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: finishing write tid 3 to mira0435601-172 2014-07-01 16:25:20,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 1 to mira0435601-412 2014-07-01 16:25:20,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 2 to mira0435601-412 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 3 to mira0435601-412 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 4 to mira0435601-412 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 6 to mira0435601-412 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: finishing write tid 4 to mira0435601-160 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 115 (ObjNum 814 snap 0 seq_num 814) dirty exists 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: finishing write tid 6 to mira0435601-311 2014-07-01 16:25:20,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 138 (ObjNum 813 snap 0 seq_num 813) dirty exists 2014-07-01 16:25:20,365.365 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 554 16 2959 2943 21.2451 48 0.068073 3.00709 2014-07-01 16:25:21,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 555 16 2959 2943 21.2068 0 - 3.00709 2014-07-01 16:25:22,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 556 16 2959 2943 21.1687 0 - 3.00709 2014-07-01 16:25:23,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 557 16 2963 2947 21.1594 5.33333 2.62199 3.00654 2014-07-01 16:25:24,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 558 16 2964 2948 21.1286 4 4.34854 3.007 2014-07-01 16:25:24,861.861 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 2, 1, 0, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:25:24,862.862 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:25:25,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: finishing write tid 4 to mira0435601-172 2014-07-01 16:25:25,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 170 (ObjNum 818 snap 0 seq_num 818) dirty exists 2014-07-01 16:25:25,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: finishing write tid 7 to mira0435601-412 2014-07-01 16:25:25,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 412 v 143 (ObjNum 819 snap 0 seq_num 819) dirty exists 2014-07-01 16:25:25,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 1 to mira0435601-394 2014-07-01 16:25:25,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 2 to mira0435601-394 2014-07-01 16:25:25,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 3 to mira0435601-394 2014-07-01 16:25:25,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 4 to mira0435601-394 2014-07-01 16:25:25,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 6 to mira0435601-394 2014-07-01 16:25:25,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: finishing write tid 7 to mira0435601-394 2014-07-01 16:25:25,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 154 (ObjNum 821 snap 0 seq_num 821) dirty exists 2014-07-01 16:25:25,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: done (6 left) 2014-07-01 16:25:25,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: done (5 left) 2014-07-01 16:25:25,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: done (4 left) 2014-07-01 16:25:25,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: done (3 left) 2014-07-01 16:25:25,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: done (2 left) 2014-07-01 16:25:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: done (1 left) 2014-07-01 16:25:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1086: done (0 left) 2014-07-01 16:25:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: read oid 283 snap -1 2014-07-01 16:25:25,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 303 current snap is 0 2014-07-01 16:25:25,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: seq_num 824 ranges {0=425984} 2014-07-01 16:25:25,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: writing mira0435601-303 from 0 to 425984 tid 1 2014-07-01 16:25:25,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1089: read oid 374 snap -1 2014-07-01 16:25:25,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: delete oid 445 current snap is 0 2014-07-01 16:25:25,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: expect (ObjNum 801 snap 0 seq_num 801) 2014-07-01 16:25:25,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 559 16 2980 2964 21.2053 64 1.11778 3.01747 2014-07-01 16:25:25,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1089: expect (ObjNum 373 snap 0 seq_num 373) 2014-07-01 16:25:25,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: finishing write tid 1 to mira0435601-303 2014-07-01 16:25:25,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: finishing write tid 3 to mira0435601-303 2014-07-01 16:25:25,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: finishing write tid 4 to mira0435601-303 2014-07-01 16:25:25,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 153 (ObjNum 824 snap 0 seq_num 824) dirty exists 2014-07-01 16:25:25,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: done (3 left) 2014-07-01 16:25:25,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: done (2 left) 2014-07-01 16:25:25,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1089: done (1 left) 2014-07-01 16:25:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: done (0 left) 2014-07-01 16:25:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 217 current snap is 0 2014-07-01 16:25:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: seq_num 826 ranges {16384=774144,790528=761856,1552384=581632,2134016=262144} 2014-07-01 16:25:25,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: writing mira0435601-217 from 16384 to 790528 tid 1 2014-07-01 16:25:25,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: writing mira0435601-217 from 790528 to 1552384 tid 2 2014-07-01 16:25:25,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: writing mira0435601-217 from 1552384 to 2134016 tid 3 2014-07-01 16:25:25,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: writing mira0435601-217 from 2134016 to 2396160 tid 4 2014-07-01 16:25:25,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 16:25:25,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: seq_num 827 ranges {344064=647168,991232=589824,1581056=409600,1990656=606208} 2014-07-01 16:25:25,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0435601-288 from 344064 to 991232 tid 1 2014-07-01 16:25:25,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0435601-288 from 991232 to 1581056 tid 2 2014-07-01 16:25:25,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0435601-288 from 1581056 to 1990656 tid 3 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0435601-288 from 1990656 to 2596864 tid 4 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: read oid 165 snap -1 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 1 to mira0435601-217 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1094: read oid 387 snap -1 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: read oid 344 snap -1 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 123 current snap is 0 2014-07-01 16:25:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: seq_num 828 ranges {0=671744,671744=401408} 2014-07-01 16:25:25,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: writing mira0435601-123 from 0 to 671744 tid 1 2014-07-01 16:25:25,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: writing mira0435601-123 from 671744 to 1073152 tid 2 2014-07-01 16:25:25,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1097: read oid 444 snap -1 2014-07-01 16:25:25,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 356 current snap is 0 2014-07-01 16:25:25,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: seq_num 829 ranges {552960=544768,1097728=401408} 2014-07-01 16:25:25,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: writing mira0435601-356 from 552960 to 1097728 tid 1 2014-07-01 16:25:25,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: writing mira0435601-356 from 1097728 to 1499136 tid 2 2014-07-01 16:25:25,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: read oid 205 snap -1 2014-07-01 16:25:25,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: expect (ObjNum 164 snap 0 seq_num 164) 2014-07-01 16:25:26,252.252 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:25:26,261.261 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:25:26,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:25:26.279482min lat: 0.057269 max lat: 48.8206 avg lat: 3.01149 2014-07-01 16:25:26,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:25:26,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 560 16 2987 2971 21.2174 28 0.699477 3.01149 2014-07-01 16:25:26,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: expect (ObjNum 204 snap 0 seq_num 204) 2014-07-01 16:25:27,238.238 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:25:27,248.248 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:25:27,366.366 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 561 16 2990 2974 21.201 12 0.705993 3.00961 2014-07-01 16:25:28,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: expect (ObjNum 343 snap 0 seq_num 343) 2014-07-01 16:25:28,247.247 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:25:28,261.261 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:25:28,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 562 16 3003 2987 21.2558 52 0.186016 3.00569 2014-07-01 16:25:28,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1097: expect (ObjNum 802 snap 0 seq_num 802) 2014-07-01 16:25:29,261.261 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:25:29,270.270 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:25:29,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 563 16 3028 3012 21.3956 100 0.498334 2.98815 2014-07-01 16:25:29,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1094: expect (ObjNum 386 snap 0 seq_num 386) 2014-07-01 16:25:29,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: finishing write tid 1 to mira0435601-356 2014-07-01 16:25:29,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: finishing write tid 2 to mira0435601-356 2014-07-01 16:25:29,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: finishing write tid 4 to mira0435601-356 2014-07-01 16:25:29,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: finishing write tid 5 to mira0435601-356 2014-07-01 16:25:29,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 356 v 84 (ObjNum 829 snap 0 seq_num 829) dirty exists 2014-07-01 16:25:29,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 1 to mira0435601-288 2014-07-01 16:25:29,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 2 to mira0435601-288 2014-07-01 16:25:29,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 3 to mira0435601-288 2014-07-01 16:25:29,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 2 to mira0435601-217 2014-07-01 16:25:29,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 3 to mira0435601-217 2014-07-01 16:25:29,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 4 to mira0435601-217 2014-07-01 16:25:29,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: done (8 left) 2014-07-01 16:25:29,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1094: done (7 left) 2014-07-01 16:25:29,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: done (6 left) 2014-07-01 16:25:29,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1097: done (5 left) 2014-07-01 16:25:29,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: done (4 left) 2014-07-01 16:25:29,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: done (3 left) 2014-07-01 16:25:29,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 96 current snap is 0 2014-07-01 16:25:29,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: seq_num 830 ranges {2363392=466944,2830336=585728,3416064=581632,3997696=36864} 2014-07-01 16:25:29,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: writing mira0435601-96 from 2363392 to 2830336 tid 1 2014-07-01 16:25:29,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: writing mira0435601-96 from 2830336 to 3416064 tid 2 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: writing mira0435601-96 from 3416064 to 3997696 tid 3 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: writing mira0435601-96 from 3997696 to 4034560 tid 4 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: delete oid 476 current snap is 0 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 6 to mira0435601-217 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 4 to mira0435601-288 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 6 to mira0435601-288 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: finishing write tid 1 to mira0435601-123 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: finishing write tid 2 to mira0435601-123 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: finishing write tid 4 to mira0435601-123 2014-07-01 16:25:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: finishing write tid 5 to mira0435601-123 2014-07-01 16:25:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 149 (ObjNum 828 snap 0 seq_num 828) dirty exists 2014-07-01 16:25:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 7 to mira0435601-288 2014-07-01 16:25:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 243 (ObjNum 827 snap 0 seq_num 827) dirty exists 2014-07-01 16:25:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: finishing write tid 7 to mira0435601-217 2014-07-01 16:25:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 146 (ObjNum 826 snap 0 seq_num 826) dirty exists 2014-07-01 16:25:30,367.367 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 564 16 3028 3012 21.3577 0 - 2.98815 2014-07-01 16:25:30,595.595 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:25:30,604.604 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:25:31,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 565 16 3034 3018 21.3624 12 2.58927 2.98559 2014-07-01 16:25:31,608.608 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:25:31,622.622 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:25:31,622.622 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:25:32,330.330 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:25:32,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 566 16 3034 3018 21.3246 0 - 2.98559 2014-07-01 16:25:32,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 1 to mira0435601-96 2014-07-01 16:25:32,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 2 to mira0435601-96 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 3 to mira0435601-96 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 4 to mira0435601-96 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 6 to mira0435601-96 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: done (4 left) 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: done (3 left) 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: done (2 left) 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: done (1 left) 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 490 current snap is 0 2014-07-01 16:25:32,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: seq_num 832 ranges {2039808=364544} 2014-07-01 16:25:32,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: writing mira0435601-490 from 2039808 to 2404352 tid 1 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1103: read oid 235 snap -1 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: finishing write tid 7 to mira0435601-96 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 171 (ObjNum 830 snap 0 seq_num 830) dirty exists 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: done (2 left) 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: read oid 284 snap -1 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1105: read oid 135 snap -1 2014-07-01 16:25:32,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1106: delete oid 104 current snap is 0 2014-07-01 16:25:32,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1106: done (4 left) 2014-07-01 16:25:32,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 459 current snap is 0 2014-07-01 16:25:32,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: seq_num 834 ranges {483328=712704,1196032=565248,1761280=630784,2392064=409600} 2014-07-01 16:25:32,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: writing mira0435601-459 from 483328 to 1196032 tid 1 2014-07-01 16:25:32,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: writing mira0435601-459 from 1196032 to 1761280 tid 2 2014-07-01 16:25:32,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: writing mira0435601-459 from 1761280 to 2392064 tid 3 2014-07-01 16:25:32,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: writing mira0435601-459 from 2392064 to 2801664 tid 4 2014-07-01 16:25:32,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: delete oid 441 current snap is 0 2014-07-01 16:25:32,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1103: expect (ObjNum 534 snap 0 seq_num 534) 2014-07-01 16:25:33,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 567 16 3041 3025 21.3364 14 2.48179 2.98717 2014-07-01 16:25:33,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1105: expect (ObjNum 704 snap 0 seq_num 704) 2014-07-01 16:25:33,719.719 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:25:34,369.369 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 568 16 3041 3025 21.2988 0 - 2.98717 2014-07-01 16:25:34,426.426 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:25:35,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: expect (ObjNum 783 snap 0 seq_num 783) 2014-07-01 16:25:35,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 569 16 3041 3025 21.2614 0 - 2.98717 2014-07-01 16:25:35,616.616 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:25:36,365.365 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.020716', u'skew': u'-0.008792', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002614', u'skew': u'-0.027337', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:25:36.108709', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884122656, u'health': u'HEALTH_OK', u'kb_used': 12658396, u'store_stats': {u'bytes_total': 6178957, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1984637}}, {u'last_updated': u'2014-07-01 07:24:54.529695', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887370536, u'health': u'HEALTH_OK', u'kb_used': 9410516, u'store_stats': {u'bytes_total': 5130449, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1984705}}, {u'last_updated': u'2014-07-01 07:25:11.312295', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887349812, u'health': u'HEALTH_OK', u'kb_used': 9431240, u'store_stats': {u'bytes_total': 6179040, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1984720}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 66 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 194, u'data_bytes': 13565703033, u'bytes_used': 28746244096, u'op_per_sec': 11, u'version': 499, u'pgs_by_state': [{u'count': 194, u'state_name': u'active+clean'}], u'read_bytes_sec': 2157402, u'write_bytes_sec': 20988453, u'bytes_avail': 5969552801792}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 263, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:25:36,365.365 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:25:36,366.366 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:25:36,366.366 INFO:teuthology.task.thrashosds.thrasher:Removing osd 4, in_osds are: [4, 2, 1, 0, 3, 5] 2014-07-01 16:25:36,367.367 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 4' 2014-07-01 16:25:36,369.369 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 570 16 3062 3046 21.3714 28 3.76132 2.99323 2014-07-01 16:25:37,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: finishing write tid 1 to mira0435601-490 2014-07-01 16:25:37,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: finishing write tid 3 to mira0435601-490 2014-07-01 16:25:37,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: finishing write tid 4 to mira0435601-490 2014-07-01 16:25:37,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 88 (ObjNum 832 snap 0 seq_num 832) dirty exists 2014-07-01 16:25:37,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 1 to mira0435601-459 2014-07-01 16:25:37,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 2 to mira0435601-459 2014-07-01 16:25:37,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 3 to mira0435601-459 2014-07-01 16:25:37,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 4 to mira0435601-459 2014-07-01 16:25:37,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 6 to mira0435601-459 2014-07-01 16:25:37,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: finishing write tid 7 to mira0435601-459 2014-07-01 16:25:37,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 186 (ObjNum 834 snap 0 seq_num 834) dirty exists 2014-07-01 16:25:37,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: done (5 left) 2014-07-01 16:25:37,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1103: done (4 left) 2014-07-01 16:25:37,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: done (3 left) 2014-07-01 16:25:37,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:1105: done (2 left) 2014-07-01 16:25:37,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: done (1 left) 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: done (0 left) 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: read oid 177 snap -1 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 162 current snap is 0 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: seq_num 836 ranges {2326528=458752,2785280=782336,3567616=577536,4145152=245760} 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0435601-162 from 2326528 to 2785280 tid 1 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0435601-162 from 2785280 to 3567616 tid 2 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0435601-162 from 3567616 to 4145152 tid 3 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0435601-162 from 4145152 to 4390912 tid 4 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1111: read oid 303 snap -1 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: read oid 403 snap -1 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: delete oid 152 current snap is 0 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: done (4 left) 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: seq_num 838 ranges {5267456=512000,5779456=794624,6574080=475136,7049216=335872} 2014-07-01 16:25:37,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: writing mira0435601-391 from 5267456 to 5779456 tid 1 2014-07-01 16:25:37,497.497 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 571 16 3072 3056 21.404 40 1.00682 2.98548 2014-07-01 16:25:37,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: writing mira0435601-391 from 5779456 to 6574080 tid 2 2014-07-01 16:25:37,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: writing mira0435601-391 from 6574080 to 7049216 tid 3 2014-07-01 16:25:37,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: writing mira0435601-391 from 7049216 to 7385088 tid 4 2014-07-01 16:25:37,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1115: delete oid 465 current snap is 0 2014-07-01 16:25:37,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1111: expect (ObjNum 824 snap 0 seq_num 824) 2014-07-01 16:25:37,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: expect (ObjNum 779 snap 0 seq_num 779) 2014-07-01 16:25:38,361.361 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.4. 2014-07-01 16:25:38,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 572 16 3088 3072 21.4785 64 1.37932 2.9758 2014-07-01 16:25:39,369.369 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 573 16 3090 3074 21.455 8 1.25825 2.97461 2014-07-01 16:25:40,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 1 to mira0435601-162 2014-07-01 16:25:40,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 2 to mira0435601-162 2014-07-01 16:25:40,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 3 to mira0435601-162 2014-07-01 16:25:40,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 4 to mira0435601-162 2014-07-01 16:25:40,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 6 to mira0435601-162 2014-07-01 16:25:40,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: expect (ObjNum 724 snap 0 seq_num 724) 2014-07-01 16:25:40,368.368 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 574 16 3093 3077 21.4385 12 2.80411 2.97422 2014-07-01 16:25:41,370.370 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 575 16 3104 3088 21.4777 44 2.95809 2.97272 2014-07-01 16:25:42,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 1 to mira0435601-391 2014-07-01 16:25:42,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 7 to mira0435601-162 2014-07-01 16:25:42,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 142 (ObjNum 836 snap 0 seq_num 836) dirty exists 2014-07-01 16:25:42,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 2 to mira0435601-391 2014-07-01 16:25:42,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 3 to mira0435601-391 2014-07-01 16:25:42,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 4 to mira0435601-391 2014-07-01 16:25:42,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 6 to mira0435601-391 2014-07-01 16:25:42,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: finishing write tid 7 to mira0435601-391 2014-07-01 16:25:42,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 175 (ObjNum 838 snap 0 seq_num 838) dirty exists 2014-07-01 16:25:42,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 576 16 3111 3095 21.489 28 0.918372 2.96821 2014-07-01 16:25:43,371.371 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:25:43,371.371 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:25:43,371.371 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2014-07-01 16:25:43,372.372 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 1' 2014-07-01 16:25:43,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 577 16 3113 3097 21.4656 8 1.48196 2.96774 2014-07-01 16:25:44,370.370 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 578 16 3118 3102 21.4631 20 2.80345 2.96914 2014-07-01 16:25:45,371.371 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 579 16 3120 3104 21.4398 8 1.6233 2.9683 2014-07-01 16:25:46,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:25:46.282884min lat: 0.057269 max lat: 48.8206 avg lat: 2.97294 2014-07-01 16:25:46,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:25:46,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 580 16 3128 3112 21.458 32 4.48508 2.97294 2014-07-01 16:25:47,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 581 16 3130 3114 21.4349 8 0.293651 2.97117 2014-07-01 16:25:47,413.413 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 1 (8655362) 2014-07-01 16:25:48,371.371 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 582 16 3130 3114 21.398 0 - 2.97117 2014-07-01 16:25:49,371.371 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 583 16 3130 3114 21.3613 0 - 2.97117 2014-07-01 16:25:50,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 584 16 3144 3128 21.4206 18.6667 3.47331 2.98131 2014-07-01 16:25:50,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: done (5 left) 2014-07-01 16:25:50,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: done (4 left) 2014-07-01 16:25:50,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1111: done (3 left) 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: done (2 left) 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: done (1 left) 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1115: done (0 left) 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: read oid 40 snap -1 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: read oid 11 snap -1 2014-07-01 16:25:50,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 225 current snap is 0 2014-07-01 16:25:50,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: seq_num 840 ranges {4603904=303104} 2014-07-01 16:25:50,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: writing mira0435601-225 from 4603904 to 4907008 tid 1 2014-07-01 16:25:50,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1119: read oid 436 snap -1 2014-07-01 16:25:50,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 206 current snap is 0 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: seq_num 841 ranges {462848=733184,1196032=593920,1789952=208896} 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: writing mira0435601-206 from 462848 to 1196032 tid 1 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: writing mira0435601-206 from 1196032 to 1789952 tid 2 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: writing mira0435601-206 from 1789952 to 1998848 tid 3 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: seq_num 842 ranges {0=45056} 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: writing mira0435601-124 from 0 to 45056 tid 1 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: read oid 314 snap -1 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: read oid 489 snap -1 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 15 current snap is 0 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: seq_num 843 ranges {659456=524288,1183744=757760,1941504=749568,2691072=98304} 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: writing mira0435601-15 from 659456 to 1183744 tid 1 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: writing mira0435601-15 from 1183744 to 1941504 tid 2 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: writing mira0435601-15 from 1941504 to 2691072 tid 3 2014-07-01 16:25:50,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: writing mira0435601-15 from 2691072 to 2789376 tid 4 2014-07-01 16:25:50,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: read oid 474 snap -1 2014-07-01 16:25:50,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 215 current snap is 0 2014-07-01 16:25:50,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: seq_num 844 ranges {2195456=798720,2994176=589824,3584000=471040,4055040=143360} 2014-07-01 16:25:50,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: writing mira0435601-215 from 2195456 to 2994176 tid 1 2014-07-01 16:25:50,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: writing mira0435601-215 from 2994176 to 3584000 tid 2 2014-07-01 16:25:50,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: writing mira0435601-215 from 3584000 to 4055040 tid 3 2014-07-01 16:25:50,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: writing mira0435601-215 from 4055040 to 4198400 tid 4 2014-07-01 16:25:50,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 458 current snap is 0 2014-07-01 16:25:50,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: seq_num 845 ranges {610304=667648,1277952=573440,1851392=630784} 2014-07-01 16:25:50,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: writing mira0435601-458 from 610304 to 1277952 tid 1 2014-07-01 16:25:50,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: writing mira0435601-458 from 1277952 to 1851392 tid 2 2014-07-01 16:25:50,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: writing mira0435601-458 from 1851392 to 2482176 tid 3 2014-07-01 16:25:50,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1128: read oid 96 snap -1 2014-07-01 16:25:50,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: expect (ObjNum 10 snap 0 seq_num 10) 2014-07-01 16:25:51,370.370 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 585 16 3149 3133 21.4182 20 0.817408 2.98107 2014-07-01 16:25:52,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: expect (ObjNum 668 snap 0 seq_num 668) 2014-07-01 16:25:52,370.370 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 586 16 3157 3141 21.4363 32 0.131233 2.9772 2014-07-01 16:25:52,421.421 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:25:52,421.421 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:25:52,421.421 INFO:teuthology.task.thrashosds.thrasher:Adding osd 4 2014-07-01 16:25:52,421.421 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 4' 2014-07-01 16:25:53,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 587 16 3158 3142 21.4065 4 0.860622 2.97652 2014-07-01 16:25:54,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1119: expect (ObjNum 812 snap 0 seq_num 812) 2014-07-01 16:25:54,370.370 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 588 16 3158 3142 21.3701 0 - 2.97652 2014-07-01 16:25:55,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 589 16 3162 3146 21.361 8 3.0133 2.97741 2014-07-01 16:25:55,473.473 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.4. 2014-07-01 16:25:55,480.480 INFO:teuthology.task.thrashosds.thrasher:Added osd 4 2014-07-01 16:25:55,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1128: expect (ObjNum 830 snap 0 seq_num 830) 2014-07-01 16:25:56,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 590 16 3168 3152 21.3655 24 5.58578 2.9782 2014-07-01 16:25:57,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 591 16 3176 3160 21.3835 32 0.217862 2.98013 2014-07-01 16:25:58,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 592 16 3176 3160 21.3474 0 - 2.98013 2014-07-01 16:25:58,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: finishing write tid 1 to mira0435601-225 2014-07-01 16:25:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: finishing write tid 3 to mira0435601-225 2014-07-01 16:25:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: finishing write tid 4 to mira0435601-225 2014-07-01 16:25:58,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 151 (ObjNum 840 snap 0 seq_num 840) dirty exists 2014-07-01 16:25:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 1 to mira0435601-206 2014-07-01 16:25:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: done (12 left) 2014-07-01 16:25:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: done (11 left) 2014-07-01 16:25:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1119: done (10 left) 2014-07-01 16:25:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: done (9 left) 2014-07-01 16:25:58,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1128: done (8 left) 2014-07-01 16:25:58,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: read oid 57 snap -1 2014-07-01 16:25:58,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 2 to mira0435601-206 2014-07-01 16:25:58,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 3 to mira0435601-206 2014-07-01 16:25:58,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 5 to mira0435601-206 2014-07-01 16:25:58,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 1 to mira0435601-15 2014-07-01 16:25:58,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 2 to mira0435601-15 2014-07-01 16:25:58,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 3 to mira0435601-15 2014-07-01 16:25:58,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 4 to mira0435601-15 2014-07-01 16:25:58,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 6 to mira0435601-15 2014-07-01 16:25:58,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 6 to mira0435601-206 2014-07-01 16:25:58,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 134 (ObjNum 841 snap 0 seq_num 841) dirty exists 2014-07-01 16:25:58,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: finishing write tid 7 to mira0435601-15 2014-07-01 16:25:58,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 100 (ObjNum 843 snap 0 seq_num 843) dirty exists 2014-07-01 16:25:58,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: expect (ObjNum 546 snap 0 seq_num 546) 2014-07-01 16:25:59,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 593 16 3181 3165 21.3451 10 4.49504 2.97925 2014-07-01 16:26:00,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: finishing write tid 1 to mira0435601-458 2014-07-01 16:26:00,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: finishing write tid 2 to mira0435601-458 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: finishing write tid 3 to mira0435601-458 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: finishing write tid 5 to mira0435601-458 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: done (8 left) 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: done (7 left) 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: done (6 left) 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: done (5 left) 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: read oid 223 snap -1 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 265 current snap is 0 2014-07-01 16:26:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: seq_num 846 ranges {2932736=696320} 2014-07-01 16:26:00,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: writing mira0435601-265 from 2932736 to 3629056 tid 1 2014-07-01 16:26:00,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: read oid 9 snap -1 2014-07-01 16:26:00,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: expect (ObjNum 691 snap 0 seq_num 691) 2014-07-01 16:26:00,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 594 16 3187 3171 21.3496 24 4.20786 2.98253 2014-07-01 16:26:00,479.479 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:26:00,480.480 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:26:00,480.480 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 16:26:00,480.480 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:26:01,084.084 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:26:01,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 595 16 3201 3185 21.4078 56 0.369937 2.97948 2014-07-01 16:26:01,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: expect (ObjNum 595 snap 0 seq_num 595) 2014-07-01 16:26:02,371.371 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 596 16 3206 3190 21.4054 20 0.758191 2.97787 2014-07-01 16:26:02,477.477 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:26:02,477.477 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pg_num 54' 2014-07-01 16:26:03,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 597 16 3206 3190 21.3696 0 - 2.97787 2014-07-01 16:26:04,000.000 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pg_num to 54 2014-07-01 16:26:04,372.372 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 598 16 3229 3213 21.4876 46 0.473154 2.97535 2014-07-01 16:26:04,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: finishing write tid 1 to mira0435601-265 2014-07-01 16:26:04,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: finishing write tid 3 to mira0435601-265 2014-07-01 16:26:04,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: expect (ObjNum 755 snap 0 seq_num 755) 2014-07-01 16:26:05,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 599 16 3234 3218 21.4852 20 0.086951 2.97151 2014-07-01 16:26:06,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:26:06.285515min lat: 0.057269 max lat: 48.8206 avg lat: 2.97102 2014-07-01 16:26:06,374.374 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:26:06,374.374 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 600 16 3237 3221 21.4693 12 2.49499 2.97102 2014-07-01 16:26:07,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 1 to mira0435601-124 2014-07-01 16:26:07,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 3 to mira0435601-124 2014-07-01 16:26:07,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: finishing write tid 4 to mira0435601-265 2014-07-01 16:26:07,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 265 v 153 (ObjNum 846 snap 0 seq_num 846) dirty exists 2014-07-01 16:26:07,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: done (7 left) 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: done (6 left) 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: done (5 left) 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: done (4 left) 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1133: read oid 247 snap -1 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 4 to mira0435601-124 2014-07-01 16:26:07,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 156 (ObjNum 842 snap 0 seq_num 842) dirty exists 2014-07-01 16:26:07,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 1 to mira0435601-215 2014-07-01 16:26:07,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: done (4 left) 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 195 current snap is 0 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: seq_num 847 ranges {598016=569344} 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: writing mira0435601-195 from 598016 to 1167360 tid 1 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1135: read oid 416 snap -1 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 2 to mira0435601-215 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 3 to mira0435601-215 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 4 to mira0435601-215 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 6 to mira0435601-215 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: finishing write tid 6 to mira0435601-458 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 146 (ObjNum 845 snap 0 seq_num 845) dirty exists 2014-07-01 16:26:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: finishing write tid 7 to mira0435601-215 2014-07-01 16:26:07,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 161 (ObjNum 844 snap 0 seq_num 844) dirty exists 2014-07-01 16:26:07,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: expect (ObjNum 39 snap 0 seq_num 39) 2014-07-01 16:26:07,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1133: expect (ObjNum 767 snap 0 seq_num 767) 2014-07-01 16:26:07,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 601 16 3237 3221 21.4336 0 - 2.97102 2014-07-01 16:26:08,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 602 16 3257 3241 21.5309 40 0.300148 2.97105 2014-07-01 16:26:08,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1135: expect (ObjNum 415 snap 0 seq_num 415) 2014-07-01 16:26:09,012.012 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:26:09,012.012 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:26:09,012.012 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 16:26:09,012.012 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:26:09,012.012 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:26:09,013.013 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 16:26:09,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 603 16 3264 3248 21.5416 28 1.33814 2.96723 2014-07-01 16:26:10,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: finishing write tid 1 to mira0435601-195 2014-07-01 16:26:10,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: finishing write tid 3 to mira0435601-195 2014-07-01 16:26:10,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: finishing write tid 4 to mira0435601-195 2014-07-01 16:26:10,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 143 (ObjNum 847 snap 0 seq_num 847) dirty exists 2014-07-01 16:26:10,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: done (5 left) 2014-07-01 16:26:10,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: done (4 left) 2014-07-01 16:26:10,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: done (3 left) 2014-07-01 16:26:10,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1133: done (2 left) 2014-07-01 16:26:10,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: done (1 left) 2014-07-01 16:26:10,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1135: done (0 left) 2014-07-01 16:26:10,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: read oid 457 snap -1 2014-07-01 16:26:10,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 270 current snap is 0 2014-07-01 16:26:10,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: seq_num 848 ranges {372736=241664} 2014-07-01 16:26:10,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: writing mira0435601-270 from 372736 to 614400 tid 1 2014-07-01 16:26:10,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: read oid 433 snap -1 2014-07-01 16:26:10,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1139: read oid 405 snap -1 2014-07-01 16:26:10,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 384 current snap is 0 2014-07-01 16:26:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: seq_num 849 ranges {1839104=114688} 2014-07-01 16:26:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: writing mira0435601-384 from 1839104 to 1953792 tid 1 2014-07-01 16:26:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1141: read oid 465 snap -1 2014-07-01 16:26:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1142: delete oid 65 current snap is 0 2014-07-01 16:26:10,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: expect (ObjNum 761 snap 0 seq_num 761) 2014-07-01 16:26:10,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 604 16 3278 3262 21.5986 56 1.87679 2.95855 2014-07-01 16:26:10,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: expect (ObjNum 432 snap 0 seq_num 432) 2014-07-01 16:26:11,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 605 16 3278 3262 21.5629 0 - 2.95855 2014-07-01 16:26:11,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: done (6 left) 2014-07-01 16:26:11,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: done (5 left) 2014-07-01 16:26:11,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1141: done (4 left) 2014-07-01 16:26:11,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1142: done (3 left) 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1143: read oid 92 snap -1 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: finishing write tid 1 to mira0435601-270 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: read oid 439 snap -1 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1145: read oid 485 snap -1 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 16:26:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: seq_num 851 ranges {0=413696,413696=565248,978944=237568} 2014-07-01 16:26:11,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: writing mira0435601-14 from 0 to 413696 tid 1 2014-07-01 16:26:11,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: writing mira0435601-14 from 413696 to 978944 tid 2 2014-07-01 16:26:11,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: writing mira0435601-14 from 978944 to 1216512 tid 3 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: seq_num 852 ranges {2387968=688128,3076096=401408} 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: writing mira0435601-327 from 2387968 to 3076096 tid 1 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: writing mira0435601-327 from 3076096 to 3477504 tid 2 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1148: read oid 433 snap -1 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: finishing write tid 3 to mira0435601-270 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1149: delete oid 182 current snap is 0 2014-07-01 16:26:11,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1148: expect (ObjNum 432 snap 0 seq_num 432) 2014-07-01 16:26:12,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 606 16 3295 3279 21.6396 34 0.179161 2.95092 2014-07-01 16:26:13,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1143: expect (ObjNum 555 snap 0 seq_num 555) 2014-07-01 16:26:13,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 607 16 3308 3292 21.6896 52 1.34722 2.94873 2014-07-01 16:26:14,374.374 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 608 16 3312 3296 21.6802 16 0.193095 2.94552 2014-07-01 16:26:14,442.442 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:26:14,442.442 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:26:14,442.442 INFO:teuthology.task.thrashosds.thrasher:Killing osd 4, live_osds are [1, 0, 3, 2, 5, 4] 2014-07-01 16:26:14,443.443 DEBUG:teuthology.task.ceph.osd.4:waiting for process to exit 2014-07-01 16:26:14,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: finishing write tid 1 to mira0435601-327 2014-07-01 16:26:14,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: finishing write tid 2 to mira0435601-327 2014-07-01 16:26:14,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: finishing write tid 4 to mira0435601-327 2014-07-01 16:26:14,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: finishing write tid 5 to mira0435601-327 2014-07-01 16:26:14,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 140 (ObjNum 852 snap 0 seq_num 852) dirty exists 2014-07-01 16:26:14,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: finishing write tid 4 to mira0435601-270 2014-07-01 16:26:14,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 270 v 92 (ObjNum 848 snap 0 seq_num 848) dirty exists 2014-07-01 16:26:14,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 1 to mira0435601-14 2014-07-01 16:26:14,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 2 to mira0435601-14 2014-07-01 16:26:14,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 3 to mira0435601-14 2014-07-01 16:26:14,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 5 to mira0435601-14 2014-07-01 16:26:14,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1139: expect (ObjNum 747 snap 0 seq_num 747) 2014-07-01 16:26:14,766.766 INFO:teuthology.task.radosbench.radosbench.0.mira043.stderr:2014-07-01 07:26:14.679129 7f8db07e3700 0 -- 10.214.133.128:0/1005625 >> 10.214.132.120:6809/8457 pipe(0x7f8dac0188e0 sd=8 :0 s=1 pgs=0 cs=0 l=1 c=0x7f8dac0277a0).fault 2014-07-01 16:26:15,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 609 16 3317 3301 21.6774 20 0.469067 2.94347 2014-07-01 16:26:15,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: finishing write tid 1 to mira0435601-384 2014-07-01 16:26:15,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: finishing write tid 3 to mira0435601-384 2014-07-01 16:26:15,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: expect (ObjNum 438 snap 0 seq_num 438) 2014-07-01 16:26:16,373.373 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 610 16 3318 3302 21.6485 4 0.096061 2.94261 2014-07-01 16:26:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: finishing write tid 4 to mira0435601-384 2014-07-01 16:26:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 193 (ObjNum 849 snap 0 seq_num 849) dirty exists 2014-07-01 16:26:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1145: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:26:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 6 to mira0435601-14 2014-07-01 16:26:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 104 (ObjNum 851 snap 0 seq_num 851) dirty exists 2014-07-01 16:26:17,374.374 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 611 16 3318 3302 21.613 0 - 2.94261 2014-07-01 16:26:18,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 612 16 3318 3302 21.5777 0 - 2.94261 2014-07-01 16:26:19,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 613 16 3327 3311 21.6012 12 5.08904 2.94904 2014-07-01 16:26:20,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 614 16 3327 3311 21.566 0 - 2.94904 2014-07-01 16:26:20,443.443 INFO:teuthology.task.ceph.osd.4:Stopped 2014-07-01 16:26:21,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: done (9 left) 2014-07-01 16:26:21,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1139: done (8 left) 2014-07-01 16:26:21,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: done (7 left) 2014-07-01 16:26:21,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1143: done (6 left) 2014-07-01 16:26:21,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: done (5 left) 2014-07-01 16:26:21,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1145: done (4 left) 2014-07-01 16:26:21,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: done (3 left) 2014-07-01 16:26:21,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: done (2 left) 2014-07-01 16:26:21,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1148: done (1 left) 2014-07-01 16:26:21,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1149: done (0 left) 2014-07-01 16:26:21,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 408 current snap is 0 2014-07-01 16:26:21,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: seq_num 854 ranges {1716224=634880} 2014-07-01 16:26:21,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: writing mira0435601-408 from 1716224 to 2351104 tid 1 2014-07-01 16:26:21,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1151: read oid 414 snap -1 2014-07-01 16:26:21,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1152: read oid 103 snap -1 2014-07-01 16:26:21,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 304 current snap is 0 2014-07-01 16:26:21,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: seq_num 855 ranges {921600=507904} 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: writing mira0435601-304 from 921600 to 1429504 tid 1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: read oid 453 snap -1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1155: read oid 306 snap -1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1156: read oid 394 snap -1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: seq_num 856 ranges {2379776=380928} 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: writing mira0435601-359 from 2379776 to 2760704 tid 1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 246 current snap is 0 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: seq_num 857 ranges {1404928=53248} 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: writing mira0435601-246 from 1404928 to 1458176 tid 1 2014-07-01 16:26:21,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: read oid 283 snap -1 2014-07-01 16:26:21,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: expect (ObjNum 786 snap 0 seq_num 786) 2014-07-01 16:26:21,374.374 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 615 16 3341 3325 21.622 28 0.240139 2.95009 2014-07-01 16:26:22,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 616 16 3349 3333 21.6389 32 0.528037 2.94424 2014-07-01 16:26:22,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: finishing write tid 1 to mira0435601-359 2014-07-01 16:26:22,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: finishing write tid 3 to mira0435601-359 2014-07-01 16:26:22,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: finishing write tid 4 to mira0435601-359 2014-07-01 16:26:22,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 148 (ObjNum 856 snap 0 seq_num 856) dirty exists 2014-07-01 16:26:22,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:1152: expect (ObjNum 725 snap 0 seq_num 725) 2014-07-01 16:26:23,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 617 16 3349 3333 21.6038 0 - 2.94424 2014-07-01 16:26:24,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 618 16 3349 3333 21.5688 0 - 2.94424 2014-07-01 16:26:25,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 619 16 3349 3333 21.534 0 - 2.94424 2014-07-01 16:26:25,446.446 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [4] live_osds: [1, 0, 3, 2, 5] 2014-07-01 16:26:25,446.446 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:26:25,446.446 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 16:26:25,446.446 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:26:25,446.446 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:26:25,446.446 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 16:26:25,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: finishing write tid 1 to mira0435601-304 2014-07-01 16:26:25,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: finishing write tid 3 to mira0435601-304 2014-07-01 16:26:25,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: finishing write tid 4 to mira0435601-304 2014-07-01 16:26:25,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 304 v 146 (ObjNum 855 snap 0 seq_num 855) dirty exists 2014-07-01 16:26:25,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1155: expect (ObjNum 305 snap 0 seq_num 305) 2014-07-01 16:26:26,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:26:26.288621min lat: 0.057269 max lat: 48.8206 avg lat: 2.93735 2014-07-01 16:26:26,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:26:26,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 620 16 3362 3346 21.5831 13 0.15564 2.93735 2014-07-01 16:26:27,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 621 16 3369 3353 21.5934 28 0.108472 2.93163 2014-07-01 16:26:27,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1152: done (9 left) 2014-07-01 16:26:27,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: done (8 left) 2014-07-01 16:26:27,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: done (7 left) 2014-07-01 16:26:27,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1155: done (6 left) 2014-07-01 16:26:27,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: done (5 left) 2014-07-01 16:26:27,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 257 current snap is 0 2014-07-01 16:26:27,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: seq_num 858 ranges {2318336=729088,3047424=745472,3792896=708608,4501504=151552} 2014-07-01 16:26:27,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: writing mira0435601-257 from 2318336 to 3047424 tid 1 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: writing mira0435601-257 from 3047424 to 3792896 tid 2 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: writing mira0435601-257 from 3792896 to 4501504 tid 3 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: writing mira0435601-257 from 4501504 to 4653056 tid 4 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1161: read oid 312 snap -1 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 166 current snap is 0 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: seq_num 859 ranges {0=602112,602112=770048,1372160=585728,1957888=249856} 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: writing mira0435601-166 from 0 to 602112 tid 1 2014-07-01 16:26:27,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: writing mira0435601-166 from 602112 to 1372160 tid 2 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: writing mira0435601-166 from 1372160 to 1957888 tid 3 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: writing mira0435601-166 from 1957888 to 2207744 tid 4 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: read oid 411 snap -1 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1161: expect (ObjNum 311 snap 0 seq_num 311) 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 1 to mira0435601-166 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 2 to mira0435601-166 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1161: done (8 left) 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 57 current snap is 0 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: seq_num 860 ranges {0=626688,626688=401408} 2014-07-01 16:26:27,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: writing mira0435601-57 from 0 to 626688 tid 1 2014-07-01 16:26:27,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: writing mira0435601-57 from 626688 to 1028096 tid 2 2014-07-01 16:26:27,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1165: delete oid 284 current snap is 0 2014-07-01 16:26:27,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 3 to mira0435601-166 2014-07-01 16:26:27,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 4 to mira0435601-166 2014-07-01 16:26:27,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 6 to mira0435601-166 2014-07-01 16:26:27,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: expect (ObjNum 601 snap 0 seq_num 601) 2014-07-01 16:26:28,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 622 16 3369 3353 21.5587 0 - 2.93163 2014-07-01 16:26:29,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: finishing write tid 1 to mira0435601-57 2014-07-01 16:26:29,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: finishing write tid 2 to mira0435601-57 2014-07-01 16:26:29,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: finishing write tid 4 to mira0435601-57 2014-07-01 16:26:29,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: finishing write tid 7 to mira0435601-166 2014-07-01 16:26:29,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 176 (ObjNum 859 snap 0 seq_num 859) dirty exists 2014-07-01 16:26:29,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: finishing write tid 5 to mira0435601-57 2014-07-01 16:26:29,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 107 (ObjNum 860 snap 0 seq_num 860) dirty exists 2014-07-01 16:26:29,375.375 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 623 16 3369 3353 21.5241 0 - 2.93163 2014-07-01 16:26:30,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 624 16 3369 3353 21.4896 0 - 2.93163 2014-07-01 16:26:30,910.910 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [4] live_osds: [1, 0, 3, 2, 5] 2014-07-01 16:26:30,910.910 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:26:30,910.910 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:26:30,910.910 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:26:30,910.910 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:26:30,910.910 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:26:31,376.376 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 625 16 3369 3353 21.4552 0 - 2.93163 2014-07-01 16:26:32,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 626 16 3369 3353 21.421 0 - 2.93163 2014-07-01 16:26:33,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 627 16 3369 3353 21.3868 0 - 2.93163 2014-07-01 16:26:34,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 628 16 3369 3353 21.3527 0 - 2.93163 2014-07-01 16:26:35,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 629 16 3369 3353 21.3188 0 - 2.93163 2014-07-01 16:26:36,336.336 INFO:teuthology.task.thrashosds.thrasher:in_osds: [2, 1, 0, 3, 5, 4] out_osds: [] dead_osds: [4] live_osds: [1, 0, 3, 2, 5] 2014-07-01 16:26:36,337.337 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 4 2014-07-01 16:26:36,337.337 INFO:teuthology.task.ceph.osd.4:Restarting daemon 2014-07-01 16:26:36,338.338 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 4' 2014-07-01 16:26:36,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 630 16 3369 3353 21.285 0 - 2.93163 2014-07-01 16:26:36,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: finishing write tid 1 to mira0435601-246 2014-07-01 16:26:36,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: finishing write tid 3 to mira0435601-246 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: done (9 left) 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: done (8 left) 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: done (7 left) 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1165: done (6 left) 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1166: read oid 21 snap -1 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1167: read oid 164 snap -1 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 384 current snap is 0 2014-07-01 16:26:36,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: seq_num 862 ranges {1953792=421888,2375680=569344,2945024=634880} 2014-07-01 16:26:36,670.670 INFO:teuthology.task.ceph.osd.4:Started 2014-07-01 16:26:36,670.670 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok dump_ops_in_flight' 2014-07-01 16:26:36,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: writing mira0435601-384 from 1953792 to 2375680 tid 1 2014-07-01 16:26:36,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: writing mira0435601-384 from 2375680 to 2945024 tid 2 2014-07-01 16:26:36,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: writing mira0435601-384 from 2945024 to 3579904 tid 3 2014-07-01 16:26:36,810.810 INFO:teuthology.task.ceph.osd.4.mira028.stdout:starting osd.4 at :/0 osd_data /var/lib/ceph/osd/ceph-4 /var/lib/ceph/osd/ceph-4/journal 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 312 current snap is 0 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: seq_num 863 ranges {8192=446464} 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: writing mira0435601-312 from 8192 to 454656 tid 1 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 166 current snap is 0 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: seq_num 864 ranges {2207744=475136,2682880=737280,3420160=466944} 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: writing mira0435601-166 from 2207744 to 2682880 tid 1 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: writing mira0435601-166 from 2682880 to 3420160 tid 2 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: writing mira0435601-166 from 3420160 to 3887104 tid 3 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 170 current snap is 0 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: seq_num 865 ranges {2269184=188416} 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: writing mira0435601-170 from 2269184 to 2457600 tid 1 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 491 current snap is 0 2014-07-01 16:26:36,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: seq_num 866 ranges {0=667648,667648=552960,1220608=651264,1871872=401408} 2014-07-01 16:26:36,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: writing mira0435601-491 from 0 to 667648 tid 1 2014-07-01 16:26:36,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: writing mira0435601-491 from 667648 to 1220608 tid 2 2014-07-01 16:26:36,873.873 INFO:teuthology.task.ceph.osd.4.mira028.stderr:2014-07-01 07:26:36.785904 7f5995533780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:26:36,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: writing mira0435601-491 from 1220608 to 1871872 tid 3 2014-07-01 16:26:36,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: writing mira0435601-491 from 1871872 to 2273280 tid 4 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1173: read oid 57 snap -1 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: finishing write tid 4 to mira0435601-246 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 117 (ObjNum 857 snap 0 seq_num 857) dirty exists 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 1 to mira0435601-408 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 3 to mira0435601-408 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 1 to mira0435601-257 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 2 to mira0435601-257 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 3 to mira0435601-257 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 4 to mira0435601-257 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 6 to mira0435601-257 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 4 to mira0435601-408 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 145 (ObjNum 854 snap 0 seq_num 854) dirty exists 2014-07-01 16:26:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1167: expect (ObjNum 163 snap 0 seq_num 163) 2014-07-01 16:26:37,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: expect (ObjNum 801 snap 0 seq_num 801) 2014-07-01 16:26:37,104.104 INFO:teuthology.orchestra.run.mira028.stderr:no valid command found; 10 closest matches: 2014-07-01 16:26:37,104.104 INFO:teuthology.orchestra.run.mira028.stderr:config show 2014-07-01 16:26:37,104.104 INFO:teuthology.orchestra.run.mira028.stderr:help 2014-07-01 16:26:37,104.104 INFO:teuthology.orchestra.run.mira028.stderr:log dump 2014-07-01 16:26:37,104.104 INFO:teuthology.orchestra.run.mira028.stderr:get_command_descriptions 2014-07-01 16:26:37,105.105 INFO:teuthology.orchestra.run.mira028.stderr:git_version 2014-07-01 16:26:37,105.105 INFO:teuthology.orchestra.run.mira028.stderr:config set [...] 2014-07-01 16:26:37,105.105 INFO:teuthology.orchestra.run.mira028.stderr:version 2014-07-01 16:26:37,105.105 INFO:teuthology.orchestra.run.mira028.stderr:2 2014-07-01 16:26:37,106.106 INFO:teuthology.orchestra.run.mira028.stderr:config get 2014-07-01 16:26:37,106.106 INFO:teuthology.orchestra.run.mira028.stderr:0 2014-07-01 16:26:37,106.106 INFO:teuthology.orchestra.run.mira028.stderr:admin_socket: invalid command 2014-07-01 16:26:37,112.112 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 4, ['dump_ops_in_flight'] 2014-07-01 16:26:37,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1151: expect (ObjNum 758 snap 0 seq_num 758) 2014-07-01 16:26:37,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 631 16 3400 3384 21.4477 12.4 0.389843 2.98241 2014-07-01 16:26:38,377.377 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 632 16 3419 3403 21.534 76 0.259357 2.96803 2014-07-01 16:26:39,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1166: expect (ObjNum 820 snap 0 seq_num 820) 2014-07-01 16:26:39,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 633 16 3419 3403 21.5 0 - 2.96803 2014-07-01 16:26:40,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: finishing write tid 1 to mira0435601-384 2014-07-01 16:26:40,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: finishing write tid 2 to mira0435601-384 2014-07-01 16:26:40,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: finishing write tid 7 to mira0435601-257 2014-07-01 16:26:40,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 257 v 92 (ObjNum 858 snap 0 seq_num 858) dirty exists 2014-07-01 16:26:40,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: done (13 left) 2014-07-01 16:26:40,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1151: done (12 left) 2014-07-01 16:26:40,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: done (11 left) 2014-07-01 16:26:40,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: done (10 left) 2014-07-01 16:26:40,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: done (9 left) 2014-07-01 16:26:40,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1166: done (8 left) 2014-07-01 16:26:40,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1167: done (7 left) 2014-07-01 16:26:40,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: delete oid 375 current snap is 0 2014-07-01 16:26:40,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: finishing write tid 3 to mira0435601-384 2014-07-01 16:26:40,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: finishing write tid 5 to mira0435601-384 2014-07-01 16:26:40,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: finishing write tid 6 to mira0435601-384 2014-07-01 16:26:40,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 198 (ObjNum 862 snap 0 seq_num 862) dirty exists 2014-07-01 16:26:40,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: finishing write tid 1 to mira0435601-312 2014-07-01 16:26:40,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: finishing write tid 3 to mira0435601-312 2014-07-01 16:26:40,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: finishing write tid 4 to mira0435601-312 2014-07-01 16:26:40,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 95 (ObjNum 863 snap 0 seq_num 863) dirty exists 2014-07-01 16:26:40,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: finishing write tid 1 to mira0435601-166 2014-07-01 16:26:40,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1156: expect (ObjNum 821 snap 0 seq_num 821) 2014-07-01 16:26:40,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 634 16 3437 3421 21.5796 36 0.493799 2.96053 2014-07-01 16:26:41,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 635 16 3441 3425 21.5708 16 0.771572 2.95775 2014-07-01 16:26:42,112.112 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok dump_ops_in_flight' 2014-07-01 16:26:42,378.378 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 636 16 3443 3427 21.5495 8 2.2961 2.95738 2014-07-01 16:26:42,586.586 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 1 to mira0435601-170 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 3 to mira0435601-170 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 4 to mira0435601-170 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 137 (ObjNum 865 snap 0 seq_num 865) dirty exists 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1156: done (7 left) 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: done (6 left) 2014-07-01 16:26:43,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: done (5 left) 2014-07-01 16:26:43,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: done (4 left) 2014-07-01 16:26:43,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: done (3 left) 2014-07-01 16:26:43,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 315 current snap is 0 2014-07-01 16:26:43,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: seq_num 868 ranges {3715072=409600,4124672=716800,4841472=704512,5545984=184320} 2014-07-01 16:26:43,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: writing mira0435601-315 from 3715072 to 4124672 tid 1 2014-07-01 16:26:43,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: writing mira0435601-315 from 4124672 to 4841472 tid 2 2014-07-01 16:26:43,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 637 16 3445 3429 21.5282 8 2.49805 2.95725 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: writing mira0435601-315 from 4841472 to 5545984 tid 3 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: writing mira0435601-315 from 5545984 to 5730304 tid 4 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: read oid 283 snap -1 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: finishing write tid 2 to mira0435601-166 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 1 to mira0435601-491 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: finishing write tid 3 to mira0435601-166 2014-07-01 16:26:43,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: finishing write tid 5 to mira0435601-166 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: finishing write tid 6 to mira0435601-166 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 180 (ObjNum 864 snap 0 seq_num 864) dirty exists 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 2 to mira0435601-491 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 3 to mira0435601-491 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 4 to mira0435601-491 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 6 to mira0435601-491 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: finishing write tid 7 to mira0435601-491 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 203 (ObjNum 866 snap 0 seq_num 866) dirty exists 2014-07-01 16:26:43,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1173: expect (ObjNum 860 snap 0 seq_num 860) 2014-07-01 16:26:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: expect (ObjNum 801 snap 0 seq_num 801) 2014-07-01 16:26:43,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 1 to mira0435601-315 2014-07-01 16:26:43,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: done (4 left) 2014-07-01 16:26:43,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: done (3 left) 2014-07-01 16:26:43,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1173: done (2 left) 2014-07-01 16:26:43,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: done (1 left) 2014-07-01 16:26:43,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 367 current snap is 0 2014-07-01 16:26:43,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: seq_num 869 ranges {2318336=438272,2756608=401408} 2014-07-01 16:26:43,999.999 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:26:44,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: writing mira0435601-367 from 2318336 to 2756608 tid 1 2014-07-01 16:26:44,010.010 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:26:44,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: writing mira0435601-367 from 2756608 to 3158016 tid 2 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: read oid 293 snap -1 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 2 to mira0435601-315 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 3 to mira0435601-315 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 4 to mira0435601-315 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 6 to mira0435601-315 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: finishing write tid 7 to mira0435601-315 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 180 (ObjNum 868 snap 0 seq_num 868) dirty exists 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: done (2 left) 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 203 current snap is 0 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: seq_num 870 ranges {0=466944,466944=778240,1245184=651264,1896448=176128} 2014-07-01 16:26:44,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: writing mira0435601-203 from 0 to 466944 tid 1 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: writing mira0435601-203 from 466944 to 1245184 tid 2 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: writing mira0435601-203 from 1245184 to 1896448 tid 3 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: writing mira0435601-203 from 1896448 to 2072576 tid 4 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1180: read oid 228 snap -1 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1181: read oid 467 snap -1 2014-07-01 16:26:44,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 372 current snap is 0 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: seq_num 871 ranges {684032=630784,1314816=557056,1871872=245760} 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: writing mira0435601-372 from 684032 to 1314816 tid 1 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: writing mira0435601-372 from 1314816 to 1871872 tid 2 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: writing mira0435601-372 from 1871872 to 2117632 tid 3 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 432 current snap is 0 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: seq_num 872 ranges {2097152=258048} 2014-07-01 16:26:44,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: writing mira0435601-432 from 2097152 to 2355200 tid 1 2014-07-01 16:26:44,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1184: read oid 331 snap -1 2014-07-01 16:26:44,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1181: expect (ObjNum 466 snap 0 seq_num 466) 2014-07-01 16:26:44,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1184: expect (ObjNum 816 snap 0 seq_num 816) 2014-07-01 16:26:44,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 638 16 3457 3441 21.5697 48 0.216807 2.95615 2014-07-01 16:26:45,030.030 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:26:45,044.044 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:26:45,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 639 16 3462 3446 21.5672 20 1.43192 2.95407 2014-07-01 16:26:46,002.002 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:26:46,015.015 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:26:46,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: expect (ObjNum 525 snap 0 seq_num 525) 2014-07-01 16:26:46,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:26:46.292075min lat: 0.057269 max lat: 48.8206 avg lat: 2.95407 2014-07-01 16:26:46,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:26:46,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 640 16 3462 3446 21.5335 0 - 2.95407 2014-07-01 16:26:47,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 641 16 3479 3463 21.606 34 0.216875 2.94987 2014-07-01 16:26:47,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: finishing write tid 1 to mira0435601-372 2014-07-01 16:26:47,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: finishing write tid 2 to mira0435601-372 2014-07-01 16:26:47,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: finishing write tid 3 to mira0435601-372 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: finishing write tid 5 to mira0435601-372 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: finishing write tid 6 to mira0435601-372 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 150 (ObjNum 871 snap 0 seq_num 871) dirty exists 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: finishing write tid 1 to mira0435601-367 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: finishing write tid 2 to mira0435601-367 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: finishing write tid 4 to mira0435601-367 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: finishing write tid 5 to mira0435601-367 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 143 (ObjNum 869 snap 0 seq_num 869) dirty exists 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 1 to mira0435601-203 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: finishing write tid 1 to mira0435601-432 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 2 to mira0435601-203 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: finishing write tid 3 to mira0435601-432 2014-07-01 16:26:47,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 3 to mira0435601-203 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: done (7 left) 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: done (6 left) 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1181: done (5 left) 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: done (4 left) 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1184: done (3 left) 2014-07-01 16:26:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1185: read oid 340 snap -1 2014-07-01 16:26:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 4 to mira0435601-203 2014-07-01 16:26:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 6 to mira0435601-203 2014-07-01 16:26:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: finishing write tid 7 to mira0435601-203 2014-07-01 16:26:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 192 (ObjNum 870 snap 0 seq_num 870) dirty exists 2014-07-01 16:26:48,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: done (3 left) 2014-07-01 16:26:48,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 377 current snap is 0 2014-07-01 16:26:48,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: seq_num 873 ranges {1998848=688128,2686976=774144,3461120=28672} 2014-07-01 16:26:48,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: writing mira0435601-377 from 1998848 to 2686976 tid 1 2014-07-01 16:26:48,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: writing mira0435601-377 from 2686976 to 3461120 tid 2 2014-07-01 16:26:48,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: writing mira0435601-377 from 3461120 to 3489792 tid 3 2014-07-01 16:26:48,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1187: delete oid 26 current snap is 0 2014-07-01 16:26:48,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1185: expect (ObjNum 800 snap 0 seq_num 800) 2014-07-01 16:26:48,056.056 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:26:48,069.069 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:26:48,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 642 16 3496 3480 21.6782 68 0.319957 2.95089 2014-07-01 16:26:48,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1180: expect (ObjNum 227 snap 0 seq_num 227) 2014-07-01 16:26:49,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 643 16 3505 3489 21.7005 36 0.910782 2.94453 2014-07-01 16:26:49,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: finishing write tid 4 to mira0435601-432 2014-07-01 16:26:49,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 83 (ObjNum 872 snap 0 seq_num 872) dirty exists 2014-07-01 16:26:49,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: finishing write tid 1 to mira0435601-377 2014-07-01 16:26:49,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: finishing write tid 2 to mira0435601-377 2014-07-01 16:26:49,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: finishing write tid 3 to mira0435601-377 2014-07-01 16:26:49,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: finishing write tid 5 to mira0435601-377 2014-07-01 16:26:49,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: finishing write tid 6 to mira0435601-377 2014-07-01 16:26:49,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 149 (ObjNum 873 snap 0 seq_num 873) dirty exists 2014-07-01 16:26:49,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1180: done (4 left) 2014-07-01 16:26:49,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: done (3 left) 2014-07-01 16:26:49,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1185: done (2 left) 2014-07-01 16:26:49,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: done (1 left) 2014-07-01 16:26:49,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1187: done (0 left) 2014-07-01 16:26:49,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: read oid 270 snap -1 2014-07-01 16:26:49,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: delete oid 244 current snap is 0 2014-07-01 16:26:49,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: expect (ObjNum 848 snap 0 seq_num 848) 2014-07-01 16:26:50,306.306 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:26:50,318.318 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:26:50,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 644 16 3507 3491 21.6792 8 2.04886 2.94401 2014-07-01 16:26:51,353.353 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:26:51,363.363 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:26:51,364.364 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:26:51,379.379 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 645 16 3508 3492 21.6518 4 2.74754 2.94396 2014-07-01 16:26:52,255.255 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:26:52,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 646 16 3515 3499 21.6616 28 0.102087 2.94293 2014-07-01 16:26:52,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: done (1 left) 2014-07-01 16:26:52,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: done (0 left) 2014-07-01 16:26:52,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1190: read oid 225 snap -1 2014-07-01 16:26:52,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: read oid 27 snap -1 2014-07-01 16:26:52,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 460 current snap is 0 2014-07-01 16:26:52,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: seq_num 876 ranges {1712128=507904,2220032=401408} 2014-07-01 16:26:52,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: writing mira0435601-460 from 1712128 to 2220032 tid 1 2014-07-01 16:26:52,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: writing mira0435601-460 from 2220032 to 2621440 tid 2 2014-07-01 16:26:52,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1193: read oid 122 snap -1 2014-07-01 16:26:52,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1194: delete oid 467 current snap is 0 2014-07-01 16:26:52,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:1190: expect (ObjNum 840 snap 0 seq_num 840) 2014-07-01 16:26:53,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 647 16 3531 3515 21.7271 64 1.36959 2.94213 2014-07-01 16:26:53,617.617 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:26:54,345.345 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:26:54,395.395 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 648 16 3533 3517 21.7059 8 1.47636 2.94129 2014-07-01 16:26:55,405.405 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 649 16 3549 3533 21.771 64 2.54155 2.9364 2014-07-01 16:26:55,582.582 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:26:56,337.337 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.020716', u'skew': u'-0.008792', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002614', u'skew': u'-0.027337', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:26:36.109137', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884090000, u'health': u'HEALTH_OK', u'kb_used': 12691052, u'store_stats': {u'bytes_total': 2913623, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2389319}}, {u'last_updated': u'2014-07-01 07:26:54.530369', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887268880, u'health': u'HEALTH_OK', u'kb_used': 9512172, u'store_stats': {u'bytes_total': 4486577, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2389409}}, {u'last_updated': u'2014-07-01 07:26:11.312680', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887289544, u'health': u'HEALTH_OK', u'kb_used': 9491508, u'store_stats': {u'bytes_total': 10373344, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1984720}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'5 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 23/8456 objects degraded (0.272%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 54 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 66 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 23, u'num_pgs': 204, u'data_bytes': 15631875537, u'degraded_total': 8456, u'bytes_used': 33416183808, u'op_per_sec': 21, u'write_bytes_sec': 65270964, u'version': 565, u'pgs_by_state': [{u'count': 199, u'state_name': u'active+clean'}, {u'count': 5, u'state_name': u'active+recovering'}], u'read_bytes_sec': 12615773, u'degraded_ratio': u'0.272', u'bytes_avail': 5964882862080}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 301, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:26:56,338.338 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:26:56,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 650 16 3567 3551 21.8483 72 0.785191 2.92889 2014-07-01 16:26:57,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: expect (ObjNum 616 snap 0 seq_num 616) 2014-07-01 16:26:57,054.054 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:26:57,055.055 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:26:57,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 651 16 3569 3553 21.827 8 0.451415 2.92742 2014-07-01 16:26:57,829.829 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:26:58,380.380 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 652 16 3574 3558 21.8242 20 1.99271 2.92545 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: finishing write tid 1 to mira0435601-460 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: finishing write tid 2 to mira0435601-460 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: finishing write tid 4 to mira0435601-460 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: finishing write tid 5 to mira0435601-460 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 98 (ObjNum 876 snap 0 seq_num 876) dirty exists 2014-07-01 16:26:58,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1193: expect (ObjNum 781 snap 0 seq_num 781) 2014-07-01 16:26:59,381.381 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 653 16 3589 3573 21.8827 60 1.12557 2.92197 2014-07-01 16:27:00,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 654 16 3604 3588 21.9409 60 1.47489 2.91477 2014-07-01 16:27:01,383.383 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 655 16 3612 3596 21.9563 32 0.994126 2.91035 2014-07-01 16:27:02,021.021 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:27:02,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1190: done (4 left) 2014-07-01 16:27:02,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: done (3 left) 2014-07-01 16:27:02,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: done (2 left) 2014-07-01 16:27:02,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1193: done (1 left) 2014-07-01 16:27:02,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1194: done (0 left) 2014-07-01 16:27:02,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 283 current snap is 0 2014-07-01 16:27:02,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: seq_num 878 ranges {282624=651264} 2014-07-01 16:27:02,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: writing mira0435601-283 from 282624 to 933888 tid 1 2014-07-01 16:27:02,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 490 current snap is 0 2014-07-01 16:27:02,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: seq_num 879 ranges {2404352=679936,3084288=520192,3604480=663552,4268032=20480} 2014-07-01 16:27:02,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: writing mira0435601-490 from 2404352 to 3084288 tid 1 2014-07-01 16:27:02,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: writing mira0435601-490 from 3084288 to 3604480 tid 2 2014-07-01 16:27:02,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: writing mira0435601-490 from 3604480 to 4268032 tid 3 2014-07-01 16:27:02,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: writing mira0435601-490 from 4268032 to 4288512 tid 4 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: read oid 443 snap -1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 199 current snap is 0 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: seq_num 880 ranges {393216=196608} 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: writing mira0435601-199 from 393216 to 589824 tid 1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1199: read oid 337 snap -1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 50 current snap is 0 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: seq_num 881 ranges {1224704=69632} 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: writing mira0435601-50 from 1224704 to 1294336 tid 1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: read oid 495 snap -1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: seq_num 882 ranges {1118208=499712,1617920=565248,2183168=237568} 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: writing mira0435601-200 from 1118208 to 1617920 tid 1 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: writing mira0435601-200 from 1617920 to 2183168 tid 2 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: writing mira0435601-200 from 2183168 to 2420736 tid 3 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 16:27:02,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: seq_num 883 ranges {2236416=770048,3006464=401408} 2014-07-01 16:27:02,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 656 16 3612 3596 21.9228 0 - 2.91035 2014-07-01 16:27:02,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: writing mira0435601-226 from 2236416 to 3006464 tid 1 2014-07-01 16:27:02,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: writing mira0435601-226 from 3006464 to 3407872 tid 2 2014-07-01 16:27:02,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1204: read oid 375 snap -1 2014-07-01 16:27:02,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: expect (ObjNum 633 snap 0 seq_num 633) 2014-07-01 16:27:03,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1199: expect (ObjNum 757 snap 0 seq_num 757) 2014-07-01 16:27:03,381.381 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 657 16 3612 3596 21.8894 0 - 2.91035 2014-07-01 16:27:04,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 658 16 3630 3614 21.9656 24 0.377239 2.91117 2014-07-01 16:27:05,389.389 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 659 16 3634 3618 21.9565 16 0.588892 2.9101 2014-07-01 16:27:06,006.006 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:27:06,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:27:06.294943min lat: 0.057269 max lat: 48.8206 avg lat: 2.90516 2014-07-01 16:27:06,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:27:06,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 660 16 3648 3632 22.0081 56 1.38677 2.90516 2014-07-01 16:27:07,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 661 16 3648 3632 21.9748 0 - 2.90516 2014-07-01 16:27:07,465.465 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:27:08,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: finishing write tid 1 to mira0435601-199 2014-07-01 16:27:08,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: finishing write tid 1 to mira0435601-50 2014-07-01 16:27:08,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: finishing write tid 3 to mira0435601-50 2014-07-01 16:27:08,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: finishing write tid 3 to mira0435601-199 2014-07-01 16:27:08,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: finishing write tid 4 to mira0435601-50 2014-07-01 16:27:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 149 (ObjNum 881 snap 0 seq_num 881) dirty exists 2014-07-01 16:27:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: finishing write tid 4 to mira0435601-199 2014-07-01 16:27:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 76 (ObjNum 880 snap 0 seq_num 880) dirty exists 2014-07-01 16:27:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 1 to mira0435601-283 2014-07-01 16:27:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 3 to mira0435601-283 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 1 to mira0435601-490 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 2 to mira0435601-490 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 3 to mira0435601-490 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 4 to mira0435601-490 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 6 to mira0435601-490 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: finishing write tid 1 to mira0435601-200 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: finishing write tid 2 to mira0435601-200 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: finishing write tid 3 to mira0435601-200 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: finishing write tid 5 to mira0435601-200 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: done (9 left) 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1199: done (8 left) 2014-07-01 16:27:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: done (7 left) 2014-07-01 16:27:08,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: done (6 left) 2014-07-01 16:27:08,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1204: done (5 left) 2014-07-01 16:27:08,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 16:27:08,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: seq_num 884 ranges {626688=692224,1318912=733184,2052096=471040} 2014-07-01 16:27:08,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: writing mira0435601-351 from 626688 to 1318912 tid 1 2014-07-01 16:27:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: writing mira0435601-351 from 1318912 to 2052096 tid 2 2014-07-01 16:27:08,258.258 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.020716', u'skew': u'-0.008792', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002614', u'skew': u'-0.027337', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:26:36.109137', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 884090000, u'health': u'HEALTH_OK', u'kb_used': 12691052, u'store_stats': {u'bytes_total': 2913623, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2389319}}, {u'last_updated': u'2014-07-01 07:26:54.530369', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887268880, u'health': u'HEALTH_OK', u'kb_used': 9512172, u'store_stats': {u'bytes_total': 4486577, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2389409}}, {u'last_updated': u'2014-07-01 07:26:11.312680', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887289544, u'health': u'HEALTH_OK', u'kb_used': 9491508, u'store_stats': {u'bytes_total': 10373344, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 1984720}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 54 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 66 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 204, u'data_bytes': 15989206561, u'bytes_used': 34491842560, u'op_per_sec': 13, u'version': 572, u'pgs_by_state': [{u'count': 204, u'state_name': u'active+clean'}], u'read_bytes_sec': 1761370, u'write_bytes_sec': 33655590, u'bytes_avail': 5963807203328}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 305, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:27:08,258.258 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:27:08,258.258 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:08,258.258 INFO:teuthology.task.thrashosds.thrasher:Removing osd 2, in_osds are: [2, 1, 0, 3, 5, 4] 2014-07-01 16:27:08,258.258 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 2' 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: writing mira0435601-351 from 2052096 to 2523136 tid 3 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1206: read oid 369 snap -1 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: finishing write tid 1 to mira0435601-226 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: finishing write tid 7 to mira0435601-490 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 93 (ObjNum 879 snap 0 seq_num 879) dirty exists 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: finishing write tid 6 to mira0435601-200 2014-07-01 16:27:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 97 (ObjNum 882 snap 0 seq_num 882) dirty exists 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 4 to mira0435601-283 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 246 (ObjNum 878 snap 0 seq_num 878) dirty exists 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: finishing write tid 2 to mira0435601-226 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: finishing write tid 4 to mira0435601-226 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: finishing write tid 5 to mira0435601-226 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 100 (ObjNum 883 snap 0 seq_num 883) dirty exists 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: done (6 left) 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: done (5 left) 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: done (4 left) 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: done (3 left) 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 174 current snap is 0 2014-07-01 16:27:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: seq_num 885 ranges {2019328=561152,2580480=561152,3141632=610304,3751936=32768} 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: writing mira0435601-174 from 2019328 to 2580480 tid 1 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: writing mira0435601-174 from 2580480 to 3141632 tid 2 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: writing mira0435601-174 from 3141632 to 3751936 tid 3 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: writing mira0435601-174 from 3751936 to 3784704 tid 4 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 222 current snap is 0 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: seq_num 886 ranges {1388544=589824} 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: writing mira0435601-222 from 1388544 to 1978368 tid 1 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 151 current snap is 0 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: seq_num 887 ranges {53248=618496,671744=724992,1396736=479232} 2014-07-01 16:27:08,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0435601-151 from 53248 to 671744 tid 1 2014-07-01 16:27:08,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0435601-151 from 671744 to 1396736 tid 2 2014-07-01 16:27:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0435601-151 from 1396736 to 1875968 tid 3 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 193 current snap is 0 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: seq_num 888 ranges {1048576=331776} 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: writing mira0435601-193 from 1048576 to 1380352 tid 1 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 105 current snap is 0 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: seq_num 889 ranges {2174976=8192} 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: writing mira0435601-105 from 2174976 to 2183168 tid 1 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 188 current snap is 0 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: seq_num 890 ranges {2678784=438272,3117056=524288,3641344=278528} 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: writing mira0435601-188 from 2678784 to 3117056 tid 1 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: writing mira0435601-188 from 3117056 to 3641344 tid 2 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: writing mira0435601-188 from 3641344 to 3919872 tid 3 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:27:08,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: seq_num 891 ranges {2011136=552960,2564096=708608,3272704=606208,3878912=290816} 2014-07-01 16:27:08,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 662 16 3662 3646 22.0262 28 2.39061 2.90344 2014-07-01 16:27:08,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: writing mira0435601-21 from 2011136 to 2564096 tid 1 2014-07-01 16:27:08,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: writing mira0435601-21 from 2564096 to 3272704 tid 2 2014-07-01 16:27:08,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: writing mira0435601-21 from 3272704 to 3878912 tid 3 2014-07-01 16:27:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: writing mira0435601-21 from 3878912 to 4169728 tid 4 2014-07-01 16:27:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 457 current snap is 0 2014-07-01 16:27:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: seq_num 892 ranges {1126400=581632,1708032=401408} 2014-07-01 16:27:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: writing mira0435601-457 from 1126400 to 1708032 tid 1 2014-07-01 16:27:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: writing mira0435601-457 from 1708032 to 2109440 tid 2 2014-07-01 16:27:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 121 current snap is 0 2014-07-01 16:27:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: seq_num 893 ranges {0=659456} 2014-07-01 16:27:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: writing mira0435601-121 from 0 to 659456 tid 1 2014-07-01 16:27:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: read oid 444 snap -1 2014-07-01 16:27:08,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: expect (ObjNum 515 snap 0 seq_num 515) 2014-07-01 16:27:09,141.141 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.2. 2014-07-01 16:27:09,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 663 16 3675 3659 22.0714 52 1.09101 2.89679 2014-07-01 16:27:09,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: expect (ObjNum 802 snap 0 seq_num 802) 2014-07-01 16:27:10,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 664 16 3675 3659 22.0381 0 - 2.89679 2014-07-01 16:27:10,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 1 to mira0435601-151 2014-07-01 16:27:10,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 1 to mira0435601-222 2014-07-01 16:27:10,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 2 to mira0435601-151 2014-07-01 16:27:10,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 3 to mira0435601-151 2014-07-01 16:27:10,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 5 to mira0435601-151 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: finishing write tid 1 to mira0435601-193 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: finishing write tid 3 to mira0435601-193 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 3 to mira0435601-222 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 1 to mira0435601-21 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 2 to mira0435601-21 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 3 to mira0435601-21 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 4 to mira0435601-21 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 6 to mira0435601-21 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: done (12 left) 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1206: done (11 left) 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: done (10 left) 2014-07-01 16:27:10,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1217: read oid 331 snap -1 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 1 to mira0435601-351 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 1 to mira0435601-121 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 3 to mira0435601-121 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 4 to mira0435601-222 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 182 (ObjNum 886 snap 0 seq_num 886) dirty exists 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 6 to mira0435601-151 2014-07-01 16:27:10,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 196 (ObjNum 887 snap 0 seq_num 887) dirty exists 2014-07-01 16:27:10,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 2 to mira0435601-351 2014-07-01 16:27:10,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 1 to mira0435601-174 2014-07-01 16:27:10,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 3 to mira0435601-351 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 5 to mira0435601-351 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 2 to mira0435601-174 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: finishing write tid 4 to mira0435601-193 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 198 (ObjNum 888 snap 0 seq_num 888) dirty exists 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 3 to mira0435601-174 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: done (10 left) 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: done (9 left) 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: done (8 left) 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1218: read oid 425 snap -1 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 4 to mira0435601-174 2014-07-01 16:27:10,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 6 to mira0435601-174 2014-07-01 16:27:10,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: finishing write tid 7 to mira0435601-21 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 148 (ObjNum 891 snap 0 seq_num 891) dirty exists 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 4 to mira0435601-121 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 200 (ObjNum 893 snap 0 seq_num 893) dirty exists 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 6 to mira0435601-351 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 250 (ObjNum 884 snap 0 seq_num 884) dirty exists 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 7 to mira0435601-174 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 208 (ObjNum 885 snap 0 seq_num 885) dirty exists 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: finishing write tid 1 to mira0435601-188 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: finishing write tid 2 to mira0435601-188 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: finishing write tid 1 to mira0435601-457 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: finishing write tid 3 to mira0435601-188 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: finishing write tid 2 to mira0435601-457 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: finishing write tid 5 to mira0435601-188 2014-07-01 16:27:10,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: finishing write tid 4 to mira0435601-457 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: finishing write tid 6 to mira0435601-188 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 96 (ObjNum 890 snap 0 seq_num 890) dirty exists 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: finishing write tid 5 to mira0435601-457 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 156 (ObjNum 892 snap 0 seq_num 892) dirty exists 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: done (8 left) 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: done (7 left) 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: done (6 left) 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: done (5 left) 2014-07-01 16:27:10,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: done (4 left) 2014-07-01 16:27:10,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: done (3 left) 2014-07-01 16:27:10,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: delete oid 371 current snap is 0 2014-07-01 16:27:10,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1218: expect (ObjNum 424 snap 0 seq_num 424) 2014-07-01 16:27:11,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 665 16 3678 3662 22.023 6 2.52564 2.89653 2014-07-01 16:27:12,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1217: expect (ObjNum 816 snap 0 seq_num 816) 2014-07-01 16:27:12,383.383 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 666 16 3689 3673 22.056 44 0.33277 2.89629 2014-07-01 16:27:13,382.382 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 667 16 3696 3680 22.0649 28 0.322165 2.89417 2014-07-01 16:27:14,156.156 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 0, 3, 5, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:14,156.156 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:14,157.157 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 16:27:14,157.157 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:27:14,157.157 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:27:14,157.157 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 16:27:14,383.383 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 668 16 3696 3680 22.0319 0 - 2.89417 2014-07-01 16:27:15,384.384 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 669 16 3699 3683 22.0169 6 2.57843 2.89408 2014-07-01 16:27:16,383.383 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 670 16 3701 3685 21.996 8 4.18531 2.89472 2014-07-01 16:27:17,384.384 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 671 16 3703 3687 21.9751 8 4.87251 2.89579 2014-07-01 16:27:18,385.385 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 672 16 3703 3687 21.9424 0 - 2.89579 2014-07-01 16:27:19,385.385 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 673 16 3716 3700 21.9871 26 4.09527 2.90351 2014-07-01 16:27:19,582.582 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 0, 3, 5, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:19,582.582 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:19,582.582 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:27:19,582.582 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:27:19,582.582 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:27:19,583.583 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:27:20,384.384 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 674 16 3717 3701 21.9604 4 0.274994 2.9028 2014-07-01 16:27:21,384.384 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 675 16 3717 3701 21.9278 0 - 2.9028 2014-07-01 16:27:22,384.384 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 676 16 3724 3708 21.9368 14 2.93477 2.90211 2014-07-01 16:27:23,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 677 16 3724 3708 21.9044 0 - 2.90211 2014-07-01 16:27:24,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 678 16 3729 3713 21.9016 10 4.80752 2.90659 2014-07-01 16:27:25,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 679 16 3729 3713 21.8693 0 - 2.90659 2014-07-01 16:27:26,385.385 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:27:26.298807min lat: 0.057269 max lat: 48.8206 avg lat: 2.9081 2014-07-01 16:27:26,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:27:26,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 680 16 3730 3714 21.8431 2 8.52416 2.9081 2014-07-01 16:27:27,168.168 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 0, 3, 5, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:27,169.169 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:27,169.169 INFO:teuthology.task.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 0, 3, 5, 4] 2014-07-01 16:27:27,170.170 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 0' 2014-07-01 16:27:27,385.385 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 681 16 3747 3731 21.9108 68 0.10659 2.91158 2014-07-01 16:27:28,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 1 to mira0435601-105 2014-07-01 16:27:28,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 3 to mira0435601-105 2014-07-01 16:27:28,228.228 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.0. 2014-07-01 16:27:28,389.389 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 682 16 3752 3736 21.908 20 1.31908 2.90948 2014-07-01 16:27:29,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 4 to mira0435601-105 2014-07-01 16:27:29,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 106 (ObjNum 889 snap 0 seq_num 889) dirty exists 2014-07-01 16:27:29,386.386 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 683 16 3753 3737 21.8818 4 9.9793 2.91137 2014-07-01 16:27:30,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 684 16 3761 3745 21.8966 32 1.9187 2.91008 2014-07-01 16:27:31,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 685 16 3764 3748 21.8821 12 1.3827 2.90879 2014-07-01 16:27:32,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 686 16 3766 3750 21.8619 8 2.67344 2.90866 2014-07-01 16:27:33,241.241 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 3, 5, 4] out_osds: [2, 0] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:33,241.241 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:33,241.241 INFO:teuthology.task.thrashosds.thrasher:Adding osd 0 2014-07-01 16:27:33,241.241 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 0' 2014-07-01 16:27:33,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 687 16 3767 3751 21.8359 4 5.32321 2.90931 2014-07-01 16:27:34,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 688 16 3767 3751 21.8041 0 - 2.90931 2014-07-01 16:27:35,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 689 16 3767 3751 21.7725 0 - 2.90931 2014-07-01 16:27:35,935.935 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.0. 2014-07-01 16:27:35,943.943 INFO:teuthology.task.thrashosds.thrasher:Added osd 0 2014-07-01 16:27:36,387.387 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 690 16 3767 3751 21.7409 0 - 2.90931 2014-07-01 16:27:37,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: done (3 left) 2014-07-01 16:27:37,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1217: done (2 left) 2014-07-01 16:27:37,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1218: done (1 left) 2014-07-01 16:27:37,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: done (0 left) 2014-07-01 16:27:37,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1220: read oid 332 snap -1 2014-07-01 16:27:37,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1221: read oid 117 snap -1 2014-07-01 16:27:37,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1222: read oid 20 snap -1 2014-07-01 16:27:37,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 432 current snap is 0 2014-07-01 16:27:37,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: seq_num 895 ranges {2355200=401408} 2014-07-01 16:27:37,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: writing mira0435601-432 from 2355200 to 2756608 tid 1 2014-07-01 16:27:37,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1224: read oid 474 snap -1 2014-07-01 16:27:37,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 122 current snap is 0 2014-07-01 16:27:37,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: seq_num 896 ranges {5111808=430080,5541888=516096,6057984=479232,6537216=208896} 2014-07-01 16:27:37,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: writing mira0435601-122 from 5111808 to 5541888 tid 1 2014-07-01 16:27:37,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: writing mira0435601-122 from 5541888 to 6057984 tid 2 2014-07-01 16:27:37,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: writing mira0435601-122 from 6057984 to 6537216 tid 3 2014-07-01 16:27:37,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: writing mira0435601-122 from 6537216 to 6746112 tid 4 2014-07-01 16:27:37,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 263 current snap is 0 2014-07-01 16:27:37,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: seq_num 897 ranges {0=700416,700416=749568,1449984=53248} 2014-07-01 16:27:37,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: writing mira0435601-263 from 0 to 700416 tid 1 2014-07-01 16:27:37,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: writing mira0435601-263 from 700416 to 1449984 tid 2 2014-07-01 16:27:37,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: writing mira0435601-263 from 1449984 to 1503232 tid 3 2014-07-01 16:27:37,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1227: read oid 377 snap -1 2014-07-01 16:27:37,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1228: read oid 206 snap -1 2014-07-01 16:27:37,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1229: delete oid 395 current snap is 0 2014-07-01 16:27:37,389.389 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 691 16 3767 3751 21.7095 0 - 2.90931 2014-07-01 16:27:37,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1220: expect (ObjNum 331 snap 0 seq_num 331) 2014-07-01 16:27:37,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1224: expect (ObjNum 668 snap 0 seq_num 668) 2014-07-01 16:27:38,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 692 16 3781 3765 21.759 11.2 5.84745 2.92989 2014-07-01 16:27:39,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 693 16 3788 3772 21.768 28 0.410529 2.93355 2014-07-01 16:27:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1227: expect (ObjNum 873 snap 0 seq_num 873) 2014-07-01 16:27:40,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 694 16 3793 3777 21.7654 20 1.20664 2.93325 2014-07-01 16:27:40,944.944 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 3, 5, 4, 0] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:40,944.944 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:40,945.945 INFO:teuthology.task.thrashosds.thrasher:Adding osd 2 2014-07-01 16:27:40,945.945 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 2' 2014-07-01 16:27:41,388.388 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 695 16 3797 3781 21.7572 16 1.45247 2.93295 2014-07-01 16:27:41,950.950 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.2. 2014-07-01 16:27:41,961.961 INFO:teuthology.task.thrashosds.thrasher:Added osd 2 2014-07-01 16:27:41,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: finishing write tid 1 to mira0435601-263 2014-07-01 16:27:41,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: finishing write tid 2 to mira0435601-263 2014-07-01 16:27:41,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: finishing write tid 3 to mira0435601-263 2014-07-01 16:27:41,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: finishing write tid 5 to mira0435601-263 2014-07-01 16:27:41,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: finishing write tid 6 to mira0435601-263 2014-07-01 16:27:41,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 153 (ObjNum 897 snap 0 seq_num 897) dirty exists 2014-07-01 16:27:42,389.389 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 696 16 3803 3787 21.7604 24 3.80623 2.93309 2014-07-01 16:27:42,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1220: done (9 left) 2014-07-01 16:27:42,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1221: done (8 left) 2014-07-01 16:27:42,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1224: done (7 left) 2014-07-01 16:27:42,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: done (6 left) 2014-07-01 16:27:42,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1227: done (5 left) 2014-07-01 16:27:42,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1229: done (4 left) 2014-07-01 16:27:42,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: read oid 39 snap -1 2014-07-01 16:27:42,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 377 current snap is 0 2014-07-01 16:27:42,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: seq_num 899 ranges {3489792=651264,4141056=401408} 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: writing mira0435601-377 from 3489792 to 4141056 tid 1 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: writing mira0435601-377 from 4141056 to 4542464 tid 2 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 116 current snap is 0 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: seq_num 900 ranges {933888=520192,1454080=401408} 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0435601-116 from 933888 to 1454080 tid 1 2014-07-01 16:27:43,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0435601-116 from 1454080 to 1855488 tid 2 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 278 current snap is 0 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: seq_num 901 ranges {1359872=794624} 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: writing mira0435601-278 from 1359872 to 2154496 tid 1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: seq_num 902 ranges {1552384=471040} 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: writing mira0435601-280 from 1552384 to 2023424 tid 1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1235: read oid 238 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: read oid 40 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: read oid 145 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 106 current snap is 0 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: seq_num 903 ranges {999424=339968} 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: writing mira0435601-106 from 999424 to 1339392 tid 1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1239: read oid 15 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1240: read oid 389 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1241: read oid 450 snap -1 2014-07-01 16:27:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:27:43,389.389 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 697 16 3808 3792 21.7578 20 1.25075 2.93283 2014-07-01 16:27:44,390.390 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 698 16 3810 3794 21.7381 8 0.669677 2.93242 2014-07-01 16:27:45,390.390 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 699 16 3810 3794 21.707 0 - 2.93242 2014-07-01 16:27:46,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:27:46.302901min lat: 0.057269 max lat: 48.8206 avg lat: 2.93436 2014-07-01 16:27:46,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:27:46,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 700 16 3813 3797 21.6932 6 5.94698 2.93436 2014-07-01 16:27:46,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 1 to mira0435601-432 2014-07-01 16:27:46,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 3 to mira0435601-432 2014-07-01 16:27:46,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: expect (ObjNum 38 snap 0 seq_num 38) 2014-07-01 16:27:46,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 4 to mira0435601-432 2014-07-01 16:27:46,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 85 (ObjNum 895 snap 0 seq_num 895) dirty exists 2014-07-01 16:27:46,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:1241: expect (ObjNum 449 snap 0 seq_num 449) 2014-07-01 16:27:46,961.961 INFO:teuthology.task.thrashosds.thrasher:in_osds: [1, 3, 5, 4, 0, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:46,961.961 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:46,961.961 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 3, 5, 4, 0, 2] 2014-07-01 16:27:46,962.962 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 16:27:47,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1240: expect (ObjNum 388 snap 0 seq_num 388) 2014-07-01 16:27:47,395.395 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 701 16 3815 3799 21.6736 8 3.39615 2.93476 2014-07-01 16:27:47,517.517 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 16:27:47,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: done (15 left) 2014-07-01 16:27:47,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: done (14 left) 2014-07-01 16:27:47,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1240: done (13 left) 2014-07-01 16:27:47,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1241: done (12 left) 2014-07-01 16:27:47,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1242: read oid 184 snap -1 2014-07-01 16:27:47,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: read oid 344 snap -1 2014-07-01 16:27:47,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 16:27:47,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: seq_num 904 ranges {2568192=212992} 2014-07-01 16:27:47,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: writing mira0435601-208 from 2568192 to 2781184 tid 1 2014-07-01 16:27:47,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: read oid 296 snap -1 2014-07-01 16:27:47,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:27:48,390.390 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 702 16 3816 3800 21.6484 4 2.58281 2.93466 2014-07-01 16:27:49,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 703 16 3816 3800 21.6176 0 - 2.93466 2014-07-01 16:27:50,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: expect (ObjNum 343 snap 0 seq_num 343) 2014-07-01 16:27:50,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 704 16 3818 3802 21.5983 4 2.05642 2.93455 2014-07-01 16:27:50,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: finishing write tid 1 to mira0435601-208 2014-07-01 16:27:50,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: finishing write tid 3 to mira0435601-208 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: finishing write tid 4 to mira0435601-208 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 253 (ObjNum 904 snap 0 seq_num 904) dirty exists 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: done (15 left) 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: done (14 left) 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 243 current snap is 0 2014-07-01 16:27:50,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: seq_num 905 ranges {2654208=638976,3293184=552960,3846144=249856} 2014-07-01 16:27:50,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: writing mira0435601-243 from 2654208 to 3293184 tid 1 2014-07-01 16:27:50,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: writing mira0435601-243 from 3293184 to 3846144 tid 2 2014-07-01 16:27:50,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: writing mira0435601-243 from 3846144 to 4096000 tid 3 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: seq_num 906 ranges {6582272=512000,7094272=540672,7634944=262144} 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: writing mira0435601-337 from 6582272 to 7094272 tid 1 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: writing mira0435601-337 from 7094272 to 7634944 tid 2 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: writing mira0435601-337 from 7634944 to 7897088 tid 3 2014-07-01 16:27:50,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:27:51,390.390 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 705 16 3821 3805 21.5847 12 0.460518 2.93347 2014-07-01 16:27:51,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: finishing write tid 1 to mira0435601-278 2014-07-01 16:27:51,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: finishing write tid 3 to mira0435601-278 2014-07-01 16:27:51,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: finishing write tid 1 to mira0435601-280 2014-07-01 16:27:51,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: expect (ObjNum 39 snap 0 seq_num 39) 2014-07-01 16:27:51,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: finishing write tid 3 to mira0435601-280 2014-07-01 16:27:51,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: done (15 left) 2014-07-01 16:27:51,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: read oid 293 snap -1 2014-07-01 16:27:51,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1222: expect (ObjNum 540 snap 0 seq_num 540) 2014-07-01 16:27:52,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 706 16 3828 3812 21.5938 28 1.81844 2.94076 2014-07-01 16:27:52,548.548 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 0, 2] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:27:52,548.548 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:27:52,548.548 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 16:27:52,549.549 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:27:53,301.301 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:27:53,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 707 16 3840 3824 21.6311 48 2.40694 2.94791 2014-07-01 16:27:54,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1235: expect (ObjNum 741 snap 0 seq_num 741) 2014-07-01 16:27:54,392.392 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 708 16 3852 3836 21.6683 48 1.58574 2.94829 2014-07-01 16:27:54,652.652 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:27:54,652.652 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pg_num 64' 2014-07-01 16:27:55,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 709 16 3856 3840 21.6603 16 1.39805 2.94646 2014-07-01 16:27:55,582.582 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pg_num to 64 2014-07-01 16:27:56,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 710 16 3856 3840 21.6298 0 - 2.94646 2014-07-01 16:27:57,392.392 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 711 16 3856 3840 21.5994 0 - 2.94646 2014-07-01 16:27:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: expect (ObjNum 675 snap 0 seq_num 675) 2014-07-01 16:27:58,392.392 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 712 16 3856 3840 21.5691 0 - 2.94646 2014-07-01 16:27:59,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: expect (ObjNum 144 snap 0 seq_num 144) 2014-07-01 16:27:59,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 713 16 3856 3840 21.5388 0 - 2.94646 2014-07-01 16:27:59,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: finishing write tid 4 to mira0435601-280 2014-07-01 16:27:59,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 204 (ObjNum 902 snap 0 seq_num 902) dirty exists 2014-07-01 16:27:59,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: finishing write tid 4 to mira0435601-278 2014-07-01 16:27:59,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 202 (ObjNum 901 snap 0 seq_num 901) dirty exists 2014-07-01 16:27:59,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: finishing write tid 1 to mira0435601-337 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: finishing write tid 2 to mira0435601-337 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: finishing write tid 3 to mira0435601-337 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: finishing write tid 5 to mira0435601-337 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: finishing write tid 6 to mira0435601-337 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 257 (ObjNum 906 snap 0 seq_num 906) dirty exists 2014-07-01 16:27:59,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1242: expect (ObjNum 183 snap 0 seq_num 183) 2014-07-01 16:28:00,391.391 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 714 16 3856 3840 21.5087 0 - 2.94646 2014-07-01 16:28:00,590.590 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 0, 2] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:00,590.590 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:00,590.590 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:28:00,590.590 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:28:00,590.590 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:28:00,590.590 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:28:00,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1239: expect (ObjNum 843 snap 0 seq_num 843) 2014-07-01 16:28:01,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 715 16 3868 3852 21.5457 8 7.21917 2.96149 2014-07-01 16:28:02,392.392 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 716 16 3873 3857 21.5435 20 0.858256 2.96442 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: finishing write tid 1 to mira0435601-377 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: finishing write tid 2 to mira0435601-377 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: finishing write tid 4 to mira0435601-377 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: finishing write tid 5 to mira0435601-377 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 156 (ObjNum 899 snap 0 seq_num 899) dirty exists 2014-07-01 16:28:02,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1228: expect (ObjNum 841 snap 0 seq_num 841) 2014-07-01 16:28:03,392.392 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 717 16 3873 3857 21.5135 0 - 2.96442 2014-07-01 16:28:03,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1222: done (15 left) 2014-07-01 16:28:03,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1228: done (14 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: done (13 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: done (12 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: done (11 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1235: done (10 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: done (9 left) 2014-07-01 16:28:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1239: done (8 left) 2014-07-01 16:28:03,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1242: done (7 left) 2014-07-01 16:28:03,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: done (6 left) 2014-07-01 16:28:03,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: done (5 left) 2014-07-01 16:28:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:1249: read oid 156 snap -1 2014-07-01 16:28:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 16:28:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: seq_num 907 ranges {3768320=434176,4202496=720896,4923392=544768,5468160=339968} 2014-07-01 16:28:03,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: writing mira0435601-94 from 3768320 to 4202496 tid 1 2014-07-01 16:28:03,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: writing mira0435601-94 from 4202496 to 4923392 tid 2 2014-07-01 16:28:03,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: writing mira0435601-94 from 4923392 to 5468160 tid 3 2014-07-01 16:28:03,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: writing mira0435601-94 from 5468160 to 5808128 tid 4 2014-07-01 16:28:03,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: delete oid 452 current snap is 0 2014-07-01 16:28:03,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: expect (ObjNum 525 snap 0 seq_num 525) 2014-07-01 16:28:04,394.394 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 718 16 3890 3874 21.5782 34 2.70082 2.96264 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: finishing write tid 1 to mira0435601-106 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 1 to mira0435601-116 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 2 to mira0435601-116 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 4 to mira0435601-116 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 5 to mira0435601-116 2014-07-01 16:28:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 95 (ObjNum 900 snap 0 seq_num 900) dirty exists 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: done (7 left) 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: done (6 left) 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: done (5 left) 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: read oid 91 snap -1 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 277 current snap is 0 2014-07-01 16:28:05,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: seq_num 909 ranges {1486848=733184} 2014-07-01 16:28:05,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 719 16 3890 3874 21.5482 0 - 2.96264 2014-07-01 16:28:05,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: writing mira0435601-277 from 1486848 to 2220032 tid 1 2014-07-01 16:28:05,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1254: read oid 104 snap -1 2014-07-01 16:28:05,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: read oid 322 snap -1 2014-07-01 16:28:05,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: seq_num 910 ranges {0=761856,761856=520192,1282048=565248,1847296=118784} 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: writing mira0435601-445 from 0 to 761856 tid 1 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: writing mira0435601-445 from 761856 to 1282048 tid 2 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: writing mira0435601-445 from 1282048 to 1847296 tid 3 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: writing mira0435601-445 from 1847296 to 1966080 tid 4 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1257: read oid 352 snap -1 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1254: done (10 left) 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: read oid 388 snap -1 2014-07-01 16:28:05,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1259: delete oid 165 current snap is 0 2014-07-01 16:28:06,059.059 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 0, 2] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:06,059.059 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:06,059.059 INFO:teuthology.task.thrashosds.thrasher:Removing osd 0, in_osds are: [3, 5, 4, 0, 2] 2014-07-01 16:28:06,059.059 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 0' 2014-07-01 16:28:06,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:28:06.306055min lat: 0.057269 max lat: 48.8206 avg lat: 2.96264 2014-07-01 16:28:06,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:28:06,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 720 16 3890 3874 21.5183 0 - 2.96264 2014-07-01 16:28:07,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 721 16 3893 3877 21.5051 4 2.78241 2.96314 2014-07-01 16:28:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: finishing write tid 3 to mira0435601-106 2014-07-01 16:28:07,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: finishing write tid 1 to mira0435601-243 2014-07-01 16:28:07,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: finishing write tid 2 to mira0435601-243 2014-07-01 16:28:07,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: finishing write tid 3 to mira0435601-243 2014-07-01 16:28:07,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: finishing write tid 5 to mira0435601-243 2014-07-01 16:28:07,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1257: expect (ObjNum 351 snap 0 seq_num 351) 2014-07-01 16:28:07,991.991 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.0. 2014-07-01 16:28:08,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 722 16 3894 3878 21.4808 4 4.30416 2.96348 2014-07-01 16:28:09,394.394 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 723 16 3894 3878 21.4511 0 - 2.96348 2014-07-01 16:28:10,394.394 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 724 16 3897 3881 21.4381 6 3.31096 2.96524 2014-07-01 16:28:11,394.394 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 725 16 3897 3881 21.4085 0 - 2.96524 2014-07-01 16:28:12,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:1249: expect (ObjNum 632 snap 0 seq_num 632) 2014-07-01 16:28:12,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 726 16 3904 3888 21.4176 14 8.03794 2.97523 2014-07-01 16:28:12,999.999 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2] out_osds: [1, 0] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:12,999.999 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:12,999.999 INFO:teuthology.task.thrashosds.thrasher:Removing osd 5, in_osds are: [3, 5, 4, 2] 2014-07-01 16:28:12,999.999 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 5' 2014-07-01 16:28:13,393.393 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 727 16 3913 3897 21.4376 36 3.61354 2.97968 2014-07-01 16:28:14,395.395 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 728 16 3919 3903 21.4411 24 1.2497 2.97722 2014-07-01 16:28:15,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 729 16 3920 3904 21.4172 4 1.95841 2.97695 2014-07-01 16:28:15,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 1 to mira0435601-94 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 2 to mira0435601-94 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 3 to mira0435601-94 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 4 to mira0435601-94 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 6 to mira0435601-94 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: finishing write tid 6 to mira0435601-243 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 154 (ObjNum 905 snap 0 seq_num 905) dirty exists 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: finishing write tid 4 to mira0435601-106 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 150 (ObjNum 903 snap 0 seq_num 903) dirty exists 2014-07-01 16:28:15,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: expect (ObjNum 321 snap 0 seq_num 321) 2014-07-01 16:28:16,394.394 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 730 16 3924 3908 21.4098 16 2.55082 2.97621 2014-07-01 16:28:16,804.804 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.5. 2014-07-01 16:28:17,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: expect (ObjNum 750 snap 0 seq_num 750) 2014-07-01 16:28:17,395.395 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 731 16 3930 3914 21.4133 24 1.28781 2.97611 2014-07-01 16:28:17,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 1 to mira0435601-445 2014-07-01 16:28:17,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 2 to mira0435601-445 2014-07-01 16:28:17,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 3 to mira0435601-445 2014-07-01 16:28:17,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 4 to mira0435601-445 2014-07-01 16:28:17,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 6 to mira0435601-445 2014-07-01 16:28:17,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: finishing write tid 7 to mira0435601-445 2014-07-01 16:28:17,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 139 (ObjNum 910 snap 0 seq_num 910) dirty exists 2014-07-01 16:28:17,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: finishing write tid 7 to mira0435601-94 2014-07-01 16:28:17,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 158 (ObjNum 907 snap 0 seq_num 907) dirty exists 2014-07-01 16:28:17,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: expect (ObjNum 90 snap 0 seq_num 90) 2014-07-01 16:28:18,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 732 16 3931 3915 21.3895 4 13.8692 2.97889 2014-07-01 16:28:19,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 1 to mira0435601-122 2014-07-01 16:28:19,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 2 to mira0435601-122 2014-07-01 16:28:19,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 3 to mira0435601-122 2014-07-01 16:28:19,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 4 to mira0435601-122 2014-07-01 16:28:19,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 6 to mira0435601-122 2014-07-01 16:28:19,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: done (11 left) 2014-07-01 16:28:19,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: done (10 left) 2014-07-01 16:28:19,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1249: done (9 left) 2014-07-01 16:28:19,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: done (8 left) 2014-07-01 16:28:19,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: done (7 left) 2014-07-01 16:28:19,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: done (6 left) 2014-07-01 16:28:19,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: done (5 left) 2014-07-01 16:28:19,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1257: done (4 left) 2014-07-01 16:28:19,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: done (3 left) 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1259: done (2 left) 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 326 current snap is 0 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: seq_num 912 ranges {143360=151552} 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: writing mira0435601-326 from 143360 to 294912 tid 1 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: read oid 426 snap -1 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: read oid 62 snap -1 2014-07-01 16:28:19,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 96 current snap is 0 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: seq_num 913 ranges {4034560=581632,4616192=761856,5378048=40960} 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: writing mira0435601-96 from 4034560 to 4616192 tid 1 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: writing mira0435601-96 from 4616192 to 5378048 tid 2 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: writing mira0435601-96 from 5378048 to 5419008 tid 3 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: seq_num 914 ranges {4022272=450560,4472832=745472,5218304=57344} 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0435601-313 from 4022272 to 4472832 tid 1 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0435601-313 from 4472832 to 5218304 tid 2 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0435601-313 from 5218304 to 5275648 tid 3 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1265: read oid 378 snap -1 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1266: read oid 432 snap -1 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1267: read oid 200 snap -1 2014-07-01 16:28:19,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1268: read oid 310 snap -1 2014-07-01 16:28:19,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 295 current snap is 0 2014-07-01 16:28:19,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: seq_num 915 ranges {1679360=724992,2404352=401408} 2014-07-01 16:28:19,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: writing mira0435601-295 from 1679360 to 2404352 tid 1 2014-07-01 16:28:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: writing mira0435601-295 from 2404352 to 2805760 tid 2 2014-07-01 16:28:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 249 current snap is 0 2014-07-01 16:28:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: seq_num 916 ranges {0=802816} 2014-07-01 16:28:19,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: writing mira0435601-249 from 0 to 802816 tid 1 2014-07-01 16:28:19,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1271: read oid 476 snap -1 2014-07-01 16:28:19,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1272: delete oid 360 current snap is 0 2014-07-01 16:28:19,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 733 16 3933 3917 21.3713 8 2.23041 2.97851 2014-07-01 16:28:20,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 734 16 3934 3918 21.3476 4 1.98939 2.97826 2014-07-01 16:28:21,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 735 16 3934 3918 21.3185 0 - 2.97826 2014-07-01 16:28:21,816.816 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:21,816.816 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:21,816.816 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2014-07-01 16:28:21,816.816 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 0' 2014-07-01 16:28:22,395.395 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 736 16 3934 3918 21.2896 0 - 2.97826 2014-07-01 16:28:22,581.581 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 0 (802) 2014-07-01 16:28:23,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: finishing write tid 7 to mira0435601-122 2014-07-01 16:28:23,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 161 (ObjNum 896 snap 0 seq_num 896) dirty exists 2014-07-01 16:28:23,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 737 16 3938 3922 21.2824 5.33333 3.87487 2.98029 2014-07-01 16:28:23,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1268: expect (ObjNum 696 snap 0 seq_num 696) 2014-07-01 16:28:24,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 738 16 3946 3930 21.2969 32 0.636486 2.97979 2014-07-01 16:28:24,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:1265: expect (ObjNum 377 snap 0 seq_num 377) 2014-07-01 16:28:24,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: expect (ObjNum 425 snap 0 seq_num 425) 2014-07-01 16:28:25,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 739 16 3946 3930 21.2681 0 - 2.97979 2014-07-01 16:28:25,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:1266: expect (ObjNum 895 snap 0 seq_num 895) 2014-07-01 16:28:26,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:28:26.309387min lat: 0.057269 max lat: 48.8206 avg lat: 2.97979 2014-07-01 16:28:26,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:28:26,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 740 16 3946 3930 21.2394 0 - 2.97979 2014-07-01 16:28:27,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 741 16 3946 3930 21.2107 0 - 2.97979 2014-07-01 16:28:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: finishing write tid 1 to mira0435601-249 2014-07-01 16:28:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: finishing write tid 3 to mira0435601-249 2014-07-01 16:28:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:28:27,591.591 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:27,591.591 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:27,591.591 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 16:28:27,592.592 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:28:28,130.130 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:28:28,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 742 16 3946 3930 21.1821 0 - 2.97979 2014-07-01 16:28:28,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: finishing write tid 4 to mira0435601-249 2014-07-01 16:28:28,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 139 (ObjNum 916 snap 0 seq_num 916) dirty exists 2014-07-01 16:28:28,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: done (14 left) 2014-07-01 16:28:28,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: done (13 left) 2014-07-01 16:28:28,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: done (12 left) 2014-07-01 16:28:28,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1265: done (11 left) 2014-07-01 16:28:28,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1266: done (10 left) 2014-07-01 16:28:28,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1268: done (9 left) 2014-07-01 16:28:28,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: done (8 left) 2014-07-01 16:28:28,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:1271: done (7 left) 2014-07-01 16:28:28,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:1272: done (6 left) 2014-07-01 16:28:28,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:1273: read oid 337 snap -1 2014-07-01 16:28:28,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 16:28:28,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: seq_num 918 ranges {1835008=544768} 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: writing mira0435601-323 from 1835008 to 2379776 tid 1 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1275: read oid 30 snap -1 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 3 current snap is 0 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: seq_num 919 ranges {1466368=573440,2039808=507904,2547712=696320} 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: writing mira0435601-3 from 1466368 to 2039808 tid 1 2014-07-01 16:28:29,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: writing mira0435601-3 from 2039808 to 2547712 tid 2 2014-07-01 16:28:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: writing mira0435601-3 from 2547712 to 3244032 tid 3 2014-07-01 16:28:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1277: read oid 369 snap -1 2014-07-01 16:28:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1278: delete oid 417 current snap is 0 2014-07-01 16:28:29,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 743 16 3946 3930 21.1536 0 - 2.97979 2014-07-01 16:28:29,693.693 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 16:28:30,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 744 16 3947 3931 21.1305 0.666667 5.90231 2.98053 2014-07-01 16:28:31,053.053 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 16:28:31,396.396 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 745 16 3947 3931 21.1022 0 - 2.98053 2014-07-01 16:28:32,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 746 16 3949 3933 21.0846 4 0.093322 2.9794 2014-07-01 16:28:32,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: finishing write tid 1 to mira0435601-295 2014-07-01 16:28:32,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: finishing write tid 2 to mira0435601-295 2014-07-01 16:28:32,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: finishing write tid 4 to mira0435601-295 2014-07-01 16:28:32,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: finishing write tid 1 to mira0435601-326 2014-07-01 16:28:32,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: finishing write tid 3 to mira0435601-326 2014-07-01 16:28:33,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: finishing write tid 5 to mira0435601-295 2014-07-01 16:28:33,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 164 (ObjNum 915 snap 0 seq_num 915) dirty exists 2014-07-01 16:28:33,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: finishing write tid 4 to mira0435601-326 2014-07-01 16:28:33,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 87 (ObjNum 912 snap 0 seq_num 912) dirty exists 2014-07-01 16:28:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1275: expect (ObjNum 649 snap 0 seq_num 649) 2014-07-01 16:28:33,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 747 16 3961 3945 21.1206 48 0.329255 2.98929 2014-07-01 16:28:34,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 748 16 3961 3945 21.0924 0 - 2.98929 2014-07-01 16:28:35,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 749 16 3961 3945 21.0642 0 - 2.98929 2014-07-01 16:28:36,063.063 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:36,063.063 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:36,064.064 INFO:teuthology.task.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.744757 2014-07-01 16:28:36,064.064 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 3 0.744757161485' 2014-07-01 16:28:36,397.397 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 750 16 3961 3945 21.0362 0 - 2.98929 2014-07-01 16:28:37,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 751 16 3961 3945 21.0081 0 - 2.98929 2014-07-01 16:28:38,398.398 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 752 16 3961 3945 20.9802 0 - 2.98929 2014-07-01 16:28:39,399.399 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 753 16 3961 3945 20.9523 0 - 2.98929 2014-07-01 16:28:40,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: finishing write tid 1 to mira0435601-277 2014-07-01 16:28:40,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: finishing write tid 3 to mira0435601-277 2014-07-01 16:28:40,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: finishing write tid 4 to mira0435601-277 2014-07-01 16:28:40,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 98 (ObjNum 909 snap 0 seq_num 909) dirty exists 2014-07-01 16:28:40,399.399 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 754 16 3971 3955 20.9776 5.71429 0.127262 3.01141 2014-07-01 16:28:40,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: done (11 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: done (10 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: done (9 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1275: done (8 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1277: done (7 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1278: done (6 left) 2014-07-01 16:28:40,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 194 current snap is 0 2014-07-01 16:28:40,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: seq_num 921 ranges {3051520=765952,3817472=720896,4538368=679936,5218304=204800} 2014-07-01 16:28:40,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: writing mira0435601-194 from 3051520 to 3817472 tid 1 2014-07-01 16:28:40,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: writing mira0435601-194 from 3817472 to 4538368 tid 2 2014-07-01 16:28:40,544.544 INFO:teuthology.orchestra.run.mira079.stderr:set osd.3 primary-affinity to 0.744757 (8488082) 2014-07-01 16:28:40,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: writing mira0435601-194 from 4538368 to 5218304 tid 3 2014-07-01 16:28:40,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: writing mira0435601-194 from 5218304 to 5423104 tid 4 2014-07-01 16:28:40,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1280: delete oid 2 current snap is 0 2014-07-01 16:28:41,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 755 16 3980 3964 20.9975 36 25.4065 3.04235 2014-07-01 16:28:42,401.401 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 756 16 3980 3964 20.9697 0 - 3.04235 2014-07-01 16:28:43,401.401 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 757 16 3980 3964 20.942 0 - 3.04235 2014-07-01 16:28:44,401.401 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 758 16 3980 3964 20.9144 0 - 3.04235 2014-07-01 16:28:45,400.400 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 759 16 3981 3965 20.8921 1 4.73031 3.04277 2014-07-01 16:28:45,555.555 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:45,555.555 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:45,556.556 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 4 to 0.334643349001 2014-07-01 16:28:45,556.556 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 0.334643349001' 2014-07-01 16:28:46,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:1267: expect (ObjNum 882 snap 0 seq_num 882) 2014-07-01 16:28:46,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:28:46.313365min lat: 0.057269 max lat: 48.8206 avg lat: 3.04672 2014-07-01 16:28:46,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:28:46,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 760 16 3986 3970 20.8909 20 5.96571 3.04672 2014-07-01 16:28:46,989.989 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 0.334643 (8219312) 2014-07-01 16:28:47,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: finishing write tid 1 to mira0435601-96 2014-07-01 16:28:47,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: finishing write tid 2 to mira0435601-96 2014-07-01 16:28:47,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: finishing write tid 3 to mira0435601-96 2014-07-01 16:28:47,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: finishing write tid 5 to mira0435601-96 2014-07-01 16:28:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: finishing write tid 6 to mira0435601-96 2014-07-01 16:28:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 186 (ObjNum 913 snap 0 seq_num 913) dirty exists 2014-07-01 16:28:47,401.401 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 761 16 3989 3973 20.8792 12 6.90175 3.04634 2014-07-01 16:28:48,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 762 16 3989 3973 20.8518 0 - 3.04634 2014-07-01 16:28:49,403.403 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 763 16 3989 3973 20.8245 0 - 3.04634 2014-07-01 16:28:50,994.994 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 764 16 3989 3973 20.7972 0 - 3.04634 2014-07-01 16:28:51,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 765 16 3989 3973 20.77 0 - 3.04634 2014-07-01 16:28:52,002.002 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:52,002.002 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:52,003.003 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:28:52,004.004 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:28:52,004.004 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:28:52,006.006 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:28:52,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 766 16 3989 3973 20.7429 0 - 3.04634 2014-07-01 16:28:53,403.403 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 767 16 3994 3978 20.742 3.33333 0.139269 3.05382 2014-07-01 16:28:54,402.402 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 768 16 3998 3982 20.7358 16 0.56553 3.05249 2014-07-01 16:28:55,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 769 16 3998 3982 20.7088 0 - 3.05249 2014-07-01 16:28:56,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 770 16 3998 3982 20.6819 0 - 3.05249 2014-07-01 16:28:56,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 1 to mira0435601-323 2014-07-01 16:28:56,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 3 to mira0435601-323 2014-07-01 16:28:57,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 771 16 4000 3984 20.6655 2.66667 3.46277 3.05475 2014-07-01 16:28:57,439.439 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:28:57,439.439 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:28:57,439.439 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 16:28:57,439.439 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:28:57,439.439 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:28:57,440.440 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 16:28:58,403.403 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 772 16 4000 3984 20.6387 0 - 3.05475 2014-07-01 16:28:59,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 773 16 4000 3984 20.612 0 - 3.05475 2014-07-01 16:29:00,403.403 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 774 16 4000 3984 20.5854 0 - 3.05475 2014-07-01 16:29:01,405.405 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 775 16 4000 3984 20.5588 0 - 3.05475 2014-07-01 16:29:02,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 4 to mira0435601-323 2014-07-01 16:29:02,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 166 (ObjNum 918 snap 0 seq_num 918) dirty exists 2014-07-01 16:29:02,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:1273: expect (ObjNum 906 snap 0 seq_num 906) 2014-07-01 16:29:02,403.403 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 776 16 4000 3984 20.5323 0 - 3.05475 2014-07-01 16:29:02,890.890 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:02,891.891 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:02,891.891 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 16:29:02,891.891 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:29:03,404.404 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 777 16 4000 3984 20.5059 0 - 3.05475 2014-07-01 16:29:03,600.600 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:29:04,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 778 16 4011 3995 20.5361 6.28571 0.380439 3.07639 2014-07-01 16:29:05,038.038 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:29:05,039.039 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 36' 2014-07-01 16:29:05,406.406 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 779 16 4011 3995 20.5097 0 - 3.07639 2014-07-01 16:29:06,231.231 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 36 2014-07-01 16:29:06,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:29:06.317383min lat: 0.057269 max lat: 48.8206 avg lat: 3.07639 2014-07-01 16:29:06,429.429 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:29:06,429.429 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 780 16 4011 3995 20.4834 0 - 3.07639 2014-07-01 16:29:07,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 781 16 4011 3995 20.4572 0 - 3.07639 2014-07-01 16:29:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 1 to mira0435601-3 2014-07-01 16:29:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 1 to mira0435601-313 2014-07-01 16:29:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 2 to mira0435601-3 2014-07-01 16:29:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 2 to mira0435601-313 2014-07-01 16:29:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 3 to mira0435601-3 2014-07-01 16:29:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 3 to mira0435601-313 2014-07-01 16:29:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 5 to mira0435601-313 2014-07-01 16:29:08,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 5 to mira0435601-3 2014-07-01 16:29:08,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: done (7 left) 2014-07-01 16:29:08,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1267: done (6 left) 2014-07-01 16:29:08,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1273: done (5 left) 2014-07-01 16:29:08,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: done (4 left) 2014-07-01 16:29:08,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1280: done (3 left) 2014-07-01 16:29:08,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 43 current snap is 0 2014-07-01 16:29:08,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: seq_num 923 ranges {2011136=16384} 2014-07-01 16:29:08,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: writing mira0435601-43 from 2011136 to 2027520 tid 1 2014-07-01 16:29:08,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 68 current snap is 0 2014-07-01 16:29:08,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: seq_num 924 ranges {2207744=536576,2744320=401408} 2014-07-01 16:29:08,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: writing mira0435601-68 from 2207744 to 2744320 tid 1 2014-07-01 16:29:08,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: writing mira0435601-68 from 2744320 to 3145728 tid 2 2014-07-01 16:29:08,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 145 current snap is 0 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: seq_num 925 ranges {1216512=614400} 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: writing mira0435601-145 from 1216512 to 1830912 tid 1 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: read oid 233 snap -1 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 6 to mira0435601-3 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 3 v 147 (ObjNum 919 snap 0 seq_num 919) dirty exists 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 1 to mira0435601-194 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 6 to mira0435601-313 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 153 (ObjNum 914 snap 0 seq_num 914) dirty exists 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 2 to mira0435601-194 2014-07-01 16:29:08,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 3 to mira0435601-194 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 4 to mira0435601-194 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 6 to mira0435601-194 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: done (6 left) 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: done (5 left) 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: read oid 268 snap -1 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1286: read oid 421 snap -1 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: read oid 45 snap -1 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1288: read oid 392 snap -1 2014-07-01 16:29:08,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:1289: delete oid 414 current snap is 0 2014-07-01 16:29:08,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 782 16 4011 3995 20.431 0 - 3.07639 2014-07-01 16:29:09,405.405 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 783 16 4011 3995 20.4049 0 - 3.07639 2014-07-01 16:29:10,406.406 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 784 16 4011 3995 20.3789 0 - 3.07639 2014-07-01 16:29:11,245.245 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:11,246.246 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:11,246.246 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 16:29:11,247.247 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:29:11,248.248 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:29:11,249.249 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 16:29:11,405.405 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 785 16 4011 3995 20.3529 0 - 3.07639 2014-07-01 16:29:12,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 786 16 4011 3995 20.3271 0 - 3.07639 2014-07-01 16:29:13,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 787 16 4011 3995 20.3012 0 - 3.07639 2014-07-01 16:29:14,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: expect (ObjNum 524 snap 0 seq_num 524) 2014-07-01 16:29:14,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 788 16 4020 4004 20.3211 3.6 10.1774 3.09443 2014-07-01 16:29:15,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 789 16 4020 4004 20.2954 0 - 3.09443 2014-07-01 16:29:15,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1286: expect (ObjNum 420 snap 0 seq_num 420) 2014-07-01 16:29:15,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: finishing write tid 1 to mira0435601-68 2014-07-01 16:29:15,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: finishing write tid 2 to mira0435601-68 2014-07-01 16:29:15,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:29:16,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 790 16 4020 4004 20.2697 0 - 3.09443 2014-07-01 16:29:16,727.727 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:16,728.728 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:16,728.728 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 16:29:16,729.729 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:29:17,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: expect (ObjNum 768 snap 0 seq_num 768) 2014-07-01 16:29:17,406.406 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 791 16 4020 4004 20.2441 0 - 3.09443 2014-07-01 16:29:17,451.451 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:29:18,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 792 16 4020 4004 20.2185 0 - 3.09443 2014-07-01 16:29:18,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: finishing write tid 4 to mira0435601-68 2014-07-01 16:29:19,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 793 16 4020 4004 20.193 0 - 3.09443 2014-07-01 16:29:20,407.407 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 794 16 4020 4004 20.1676 0 - 3.09443 2014-07-01 16:29:21,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 795 16 4020 4004 20.1422 0 - 3.09443 2014-07-01 16:29:22,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 796 16 4020 4004 20.1169 0 - 3.09443 2014-07-01 16:29:23,409.409 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 797 16 4020 4004 20.0917 0 - 3.09443 2014-07-01 16:29:23,806.806 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:23,806.806 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:23,806.806 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.685012 2014-07-01 16:29:23,806.806 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0.685012409159' 2014-07-01 16:29:24,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1288: expect (ObjNum 391 snap 0 seq_num 391) 2014-07-01 16:29:24,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 798 16 4030 4014 20.1166 4 10.0902 3.12519 2014-07-01 16:29:24,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: finishing write tid 5 to mira0435601-68 2014-07-01 16:29:24,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 88 (ObjNum 924 snap 0 seq_num 924) dirty exists 2014-07-01 16:29:24,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: finishing write tid 1 to mira0435601-43 2014-07-01 16:29:24,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: finishing write tid 3 to mira0435601-43 2014-07-01 16:29:25,237.237 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0.685012 (8448922) 2014-07-01 16:29:25,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 799 16 4030 4014 20.0914 0 - 3.12519 2014-07-01 16:29:26,409.409 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:29:26.321216min lat: 0.057269 max lat: 48.8206 avg lat: 3.12519 2014-07-01 16:29:26,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:29:26,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 800 16 4030 4014 20.0663 0 - 3.12519 2014-07-01 16:29:26,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: finishing write tid 4 to mira0435601-43 2014-07-01 16:29:26,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 109 (ObjNum 923 snap 0 seq_num 923) dirty exists 2014-07-01 16:29:27,408.408 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 801 16 4037 4021 20.0762 9.33333 40.624 3.13625 2014-07-01 16:29:28,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 802 16 4037 4021 20.0512 0 - 3.13625 2014-07-01 16:29:29,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 803 16 4037 4021 20.0262 0 - 3.13625 2014-07-01 16:29:30,246.246 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2] out_osds: [1, 0, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:30,246.246 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:30,247.247 INFO:teuthology.task.thrashosds.thrasher:Adding osd 0 2014-07-01 16:29:30,247.247 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 0' 2014-07-01 16:29:30,409.409 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 804 16 4037 4021 20.0013 0 - 3.13625 2014-07-01 16:29:31,382.382 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.0. 2014-07-01 16:29:31,391.391 INFO:teuthology.task.thrashosds.thrasher:Added osd 0 2014-07-01 16:29:31,411.411 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 805 16 4043 4027 20.0063 6 50.6469 3.15266 2014-07-01 16:29:32,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 806 16 4043 4027 19.9814 0 - 3.15266 2014-07-01 16:29:33,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 807 16 4045 4029 19.9666 4 6.38272 3.15321 2014-07-01 16:29:34,411.411 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 808 16 4046 4030 19.9468 4 2.69458 3.15309 2014-07-01 16:29:35,411.411 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 809 16 4046 4030 19.9222 0 - 3.15309 2014-07-01 16:29:36,391.391 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0] out_osds: [1, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:36,391.391 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:36,392.392 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:29:36,392.392 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:29:36,392.392 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:29:36,393.393 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:29:36,410.410 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 810 16 4046 4030 19.8976 0 - 3.15309 2014-07-01 16:29:37,411.411 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 811 16 4051 4035 19.8977 6.66667 0.486752 3.15703 2014-07-01 16:29:38,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 812 16 4052 4036 19.8781 4 0.401567 3.15634 2014-07-01 16:29:39,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 813 16 4052 4036 19.8537 0 - 3.15634 2014-07-01 16:29:40,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: finishing write tid 7 to mira0435601-194 2014-07-01 16:29:40,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 210 (ObjNum 921 snap 0 seq_num 921) dirty exists 2014-07-01 16:29:40,411.411 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 814 16 4054 4038 19.8391 4 0.491666 3.15568 2014-07-01 16:29:41,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 815 16 4056 4040 19.8246 8 61.0017 3.16946 2014-07-01 16:29:41,925.925 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0] out_osds: [1, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:41,925.925 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:41,925.925 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 16:29:41,926.926 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:29:42,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: finishing write tid 1 to mira0435601-145 2014-07-01 16:29:42,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: finishing write tid 3 to mira0435601-145 2014-07-01 16:29:42,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 816 16 4067 4051 19.8542 44 0.845061 3.19312 2014-07-01 16:29:43,691.691 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 817 16 4069 4053 19.8397 8 0.943376 3.19206 2014-07-01 16:29:43,697.697 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:29:44,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 818 16 4071 4055 19.8252 8 2.25141 3.19161 2014-07-01 16:29:45,081.081 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:29:45,081.081 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pg_num 34' 2014-07-01 16:29:45,412.412 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 819 16 4074 4058 19.8156 12 3.14908 3.19141 2014-07-01 16:29:46,316.316 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pg_num to 34 2014-07-01 16:29:46,413.413 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:29:46.325741min lat: 0.057269 max lat: 61.0017 avg lat: 3.19141 2014-07-01 16:29:46,413.413 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:29:46,413.413 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 820 16 4074 4058 19.7915 0 - 3.19141 2014-07-01 16:29:47,414.414 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 821 16 4074 4058 19.7674 0 - 3.19141 2014-07-01 16:29:48,414.414 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 822 16 4074 4058 19.7433 0 - 3.19141 2014-07-01 16:29:48,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: finishing write tid 4 to mira0435601-145 2014-07-01 16:29:48,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 78 (ObjNum 925 snap 0 seq_num 925) dirty exists 2014-07-01 16:29:49,414.414 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 823 16 4083 4067 19.763 9 68.3165 3.21653 2014-07-01 16:29:50,414.414 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 824 16 4085 4069 19.7488 8 0.950194 3.21542 2014-07-01 16:29:51,324.324 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0] out_osds: [1, 5] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:51,325.325 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:51,325.325 INFO:teuthology.task.thrashosds.thrasher:Adding osd 5 2014-07-01 16:29:51,325.325 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 5' 2014-07-01 16:29:51,413.413 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 825 16 4085 4069 19.7248 0 - 3.21542 2014-07-01 16:29:52,415.415 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 826 16 4085 4069 19.701 0 - 3.21542 2014-07-01 16:29:53,414.414 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 827 16 4085 4069 19.6771 0 - 3.21542 2014-07-01 16:29:53,896.896 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.5. 2014-07-01 16:29:53,909.909 INFO:teuthology.task.thrashosds.thrasher:Added osd 5 2014-07-01 16:29:54,415.415 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 828 16 4085 4069 19.6534 0 - 3.21542 2014-07-01 16:29:55,415.415 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 829 16 4095 4079 19.6779 8 0.356649 3.22365 2014-07-01 16:29:56,415.415 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 830 16 4100 4084 19.6783 20 0.27982 3.22032 2014-07-01 16:29:57,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 831 16 4100 4084 19.6546 0 - 3.22032 2014-07-01 16:29:58,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: done (9 left) 2014-07-01 16:29:58,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: done (8 left) 2014-07-01 16:29:58,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: done (7 left) 2014-07-01 16:29:58,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: done (6 left) 2014-07-01 16:29:58,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: done (5 left) 2014-07-01 16:29:58,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: done (4 left) 2014-07-01 16:29:58,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:1286: done (3 left) 2014-07-01 16:29:58,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: done (2 left) 2014-07-01 16:29:58,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1288: done (1 left) 2014-07-01 16:29:58,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1289: done (0 left) 2014-07-01 16:29:58,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1290: read oid 328 snap -1 2014-07-01 16:29:58,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1291: read oid 480 snap -1 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1292: read oid 117 snap -1 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1293: read oid 181 snap -1 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: read oid 71 snap -1 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: seq_num 927 ranges {4227072=356352} 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: writing mira0435601-58 from 4227072 to 4583424 tid 1 2014-07-01 16:29:58,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: read oid 410 snap -1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: read oid 494 snap -1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: read oid 454 snap -1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1299: read oid 440 snap -1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1300: read oid 358 snap -1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: seq_num 928 ranges {1708032=782336,2490368=700416,3190784=102400} 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: writing mira0435601-434 from 1708032 to 2490368 tid 1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: writing mira0435601-434 from 2490368 to 3190784 tid 2 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: writing mira0435601-434 from 3190784 to 3293184 tid 3 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 129 current snap is 0 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: seq_num 929 ranges {3153920=704512,3858432=528384,4386816=729088,5115904=348160} 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: writing mira0435601-129 from 3153920 to 3858432 tid 1 2014-07-01 16:29:58,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: writing mira0435601-129 from 3858432 to 4386816 tid 2 2014-07-01 16:29:58,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: writing mira0435601-129 from 4386816 to 5115904 tid 3 2014-07-01 16:29:58,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: writing mira0435601-129 from 5115904 to 5464064 tid 4 2014-07-01 16:29:58,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 204 current snap is 0 2014-07-01 16:29:58,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: seq_num 930 ranges {1626112=577536,2203648=512000,2715648=552960,3268608=540672} 2014-07-01 16:29:58,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: writing mira0435601-204 from 1626112 to 2203648 tid 1 2014-07-01 16:29:58,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: writing mira0435601-204 from 2203648 to 2715648 tid 2 2014-07-01 16:29:58,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: writing mira0435601-204 from 2715648 to 3268608 tid 3 2014-07-01 16:29:58,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: writing mira0435601-204 from 3268608 to 3809280 tid 4 2014-07-01 16:29:58,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1304: read oid 275 snap -1 2014-07-01 16:29:58,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: read oid 248 snap -1 2014-07-01 16:29:58,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: expect (ObjNum 738 snap 0 seq_num 738) 2014-07-01 16:29:58,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 832 16 4101 4085 19.6358 2 2.6722 3.22019 2014-07-01 16:29:58,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: done (15 left) 2014-07-01 16:29:58,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 123 current snap is 0 2014-07-01 16:29:58,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: seq_num 931 ranges {1073152=602112,1675264=401408} 2014-07-01 16:29:58,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: writing mira0435601-123 from 1073152 to 1675264 tid 1 2014-07-01 16:29:58,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: writing mira0435601-123 from 1675264 to 2076672 tid 2 2014-07-01 16:29:58,910.910 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:29:58,910.910 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:29:58,910.910 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 0 to 0.810537590944 2014-07-01 16:29:58,910.910 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 0.810537590944' 2014-07-01 16:29:59,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:29:59,415.415 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 833 16 4102 4086 19.617 4 3.14976 3.22017 2014-07-01 16:29:59,968.968 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 0.810538 (8531192) 2014-07-01 16:30:00,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 834 16 4107 4091 19.6175 20 4.85125 3.22785 2014-07-01 16:30:01,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 835 16 4107 4091 19.594 0 - 3.22785 2014-07-01 16:30:02,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 836 16 4107 4091 19.5705 0 - 3.22785 2014-07-01 16:30:02,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1292: done (15 left) 2014-07-01 16:30:02,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 46 current snap is 0 2014-07-01 16:30:02,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: seq_num 932 ranges {1937408=475136,2412544=401408} 2014-07-01 16:30:02,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: writing mira0435601-46 from 1937408 to 2412544 tid 1 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: writing mira0435601-46 from 2412544 to 2813952 tid 2 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: done (15 left) 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 423 current snap is 0 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: seq_num 933 ranges {3481600=749568} 2014-07-01 16:30:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: writing mira0435601-423 from 3481600 to 4231168 tid 1 2014-07-01 16:30:02,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:30:02,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1291: done (15 left) 2014-07-01 16:30:02,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 220 current snap is 0 2014-07-01 16:30:02,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: seq_num 934 ranges {1843200=421888} 2014-07-01 16:30:02,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: writing mira0435601-220 from 1843200 to 2265088 tid 1 2014-07-01 16:30:02,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:30:02,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 1 to mira0435601-129 2014-07-01 16:30:03,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1290: expect (ObjNum 604 snap 0 seq_num 604) 2014-07-01 16:30:03,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 837 16 4118 4102 19.5997 14.6667 31.9355 3.2481 2014-07-01 16:30:04,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 838 16 4121 4105 19.5906 12 0.455222 3.24629 2014-07-01 16:30:04,976.976 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:30:04,977.977 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:30:04,977.977 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 16:30:04,979.979 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:05,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 839 16 4121 4105 19.5673 0 - 3.24629 2014-07-01 16:30:05,586.586 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:06,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:30:06.329641min lat: 0.057269 max lat: 68.3165 avg lat: 3.24625 2014-07-01 16:30:06,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:30:06,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 840 16 4124 4108 19.5583 6 3.50634 3.24625 2014-07-01 16:30:07,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:1300: expect (ObjNum 357 snap 0 seq_num 357) 2014-07-01 16:30:07,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1293: expect (ObjNum 180 snap 0 seq_num 180) 2014-07-01 16:30:07,296.296 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 16:30:07,419.419 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 841 16 4128 4112 19.554 16 3.70925 3.24661 2014-07-01 16:30:08,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: finishing write tid 1 to mira0435601-423 2014-07-01 16:30:08,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: finishing write tid 3 to mira0435601-423 2014-07-01 16:30:08,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 2 to mira0435601-129 2014-07-01 16:30:08,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 3 to mira0435601-129 2014-07-01 16:30:08,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 4 to mira0435601-129 2014-07-01 16:30:08,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 1 to mira0435601-220 2014-07-01 16:30:08,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 3 to mira0435601-220 2014-07-01 16:30:08,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: finishing write tid 1 to mira0435601-123 2014-07-01 16:30:08,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: finishing write tid 2 to mira0435601-123 2014-07-01 16:30:08,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: finishing write tid 4 to mira0435601-123 2014-07-01 16:30:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 6 to mira0435601-129 2014-07-01 16:30:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: finishing write tid 4 to mira0435601-423 2014-07-01 16:30:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 160 (ObjNum 933 snap 0 seq_num 933) dirty exists 2014-07-01 16:30:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 4 to mira0435601-220 2014-07-01 16:30:08,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 168 (ObjNum 934 snap 0 seq_num 934) dirty exists 2014-07-01 16:30:08,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:1304: expect (ObjNum 274 snap 0 seq_num 274) 2014-07-01 16:30:08,416.416 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 842 16 4138 4122 19.5783 40 0.390291 3.24413 2014-07-01 16:30:08,750.750 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 16:30:09,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 843 16 4138 4122 19.5551 0 - 3.24413 2014-07-01 16:30:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: finishing write tid 5 to mira0435601-123 2014-07-01 16:30:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 164 (ObjNum 931 snap 0 seq_num 931) dirty exists 2014-07-01 16:30:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 7 to mira0435601-129 2014-07-01 16:30:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 191 (ObjNum 929 snap 0 seq_num 929) dirty exists 2014-07-01 16:30:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: finishing write tid 1 to mira0435601-434 2014-07-01 16:30:09,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:1299: expect (ObjNum 439 snap 0 seq_num 439) 2014-07-01 16:30:09,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: finishing write tid 2 to mira0435601-434 2014-07-01 16:30:09,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: finishing write tid 3 to mira0435601-434 2014-07-01 16:30:09,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: finishing write tid 5 to mira0435601-434 2014-07-01 16:30:09,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 1 to mira0435601-204 2014-07-01 16:30:09,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 2 to mira0435601-204 2014-07-01 16:30:09,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 3 to mira0435601-204 2014-07-01 16:30:09,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 4 to mira0435601-204 2014-07-01 16:30:09,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 6 to mira0435601-204 2014-07-01 16:30:09,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: expect (ObjNum 247 snap 0 seq_num 247) 2014-07-01 16:30:09,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: expect (ObjNum 70 snap 0 seq_num 70) 2014-07-01 16:30:10,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 844 16 4139 4123 19.5367 2 1.88493 3.2438 2014-07-01 16:30:11,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: finishing write tid 6 to mira0435601-434 2014-07-01 16:30:11,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 143 (ObjNum 928 snap 0 seq_num 928) dirty exists 2014-07-01 16:30:11,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: finishing write tid 7 to mira0435601-204 2014-07-01 16:30:11,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 163 (ObjNum 930 snap 0 seq_num 930) dirty exists 2014-07-01 16:30:11,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1290: done (15 left) 2014-07-01 16:30:11,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1293: done (14 left) 2014-07-01 16:30:11,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: done (13 left) 2014-07-01 16:30:11,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:1299: done (12 left) 2014-07-01 16:30:11,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1300: done (11 left) 2014-07-01 16:30:11,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: done (10 left) 2014-07-01 16:30:11,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: done (9 left) 2014-07-01 16:30:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: done (8 left) 2014-07-01 16:30:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1304: done (7 left) 2014-07-01 16:30:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: done (6 left) 2014-07-01 16:30:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: done (5 left) 2014-07-01 16:30:11,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: done (4 left) 2014-07-01 16:30:11,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: done (3 left) 2014-07-01 16:30:11,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1310: read oid 410 snap -1 2014-07-01 16:30:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 322 current snap is 0 2014-07-01 16:30:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: seq_num 935 ranges {1900544=294912} 2014-07-01 16:30:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: writing mira0435601-322 from 1900544 to 2195456 tid 1 2014-07-01 16:30:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1312: read oid 269 snap -1 2014-07-01 16:30:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1313: delete oid 363 current snap is 0 2014-07-01 16:30:11,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1310: expect (ObjNum 738 snap 0 seq_num 738) 2014-07-01 16:30:11,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 845 16 4139 4123 19.5135 0 - 3.2438 2014-07-01 16:30:12,417.417 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 846 16 4139 4123 19.4905 0 - 3.2438 2014-07-01 16:30:13,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 847 16 4139 4123 19.4675 0 - 3.2438 2014-07-01 16:30:13,759.759 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:30:13,759.759 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:30:13,760.760 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 16:30:13,760.760 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:14,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 848 16 4145 4129 19.4728 6 6.25387 3.2503 2014-07-01 16:30:14,514.514 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:15,419.419 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 849 16 4146 4130 19.4546 4 4.59879 3.25062 2014-07-01 16:30:16,054.054 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:30:16,055.055 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 76' 2014-07-01 16:30:16,418.418 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 850 16 4148 4132 19.4411 8 7.37898 3.25269 2014-07-01 16:30:16,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: expect (ObjNum 453 snap 0 seq_num 453) 2014-07-01 16:30:17,065.065 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 76 2014-07-01 16:30:17,419.419 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 851 16 4153 4137 19.4418 20 2.28763 3.25259 2014-07-01 16:30:18,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 852 16 4157 4141 19.4377 16 3.13228 3.2522 2014-07-01 16:30:19,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 853 16 4160 4144 19.429 12 2.16189 3.25397 2014-07-01 16:30:20,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 854 16 4163 4147 19.4203 12 1.07242 3.25288 2014-07-01 16:30:21,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 855 16 4163 4147 19.3976 0 - 3.25288 2014-07-01 16:30:22,075.075 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:30:22,076.076 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:30:22,076.076 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 16:30:22,076.076 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:22,420.420 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 856 16 4163 4147 19.3749 0 - 3.25288 2014-07-01 16:30:22,886.886 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:23,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 857 16 4163 4147 19.3523 0 - 3.25288 2014-07-01 16:30:24,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 858 16 4171 4155 19.367 8 0.467641 3.25605 2014-07-01 16:30:25,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 859 16 4175 4159 19.3631 16 1.51205 3.25676 2014-07-01 16:30:26,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:30:26.333743min lat: 0.057269 max lat: 68.3165 avg lat: 3.25676 2014-07-01 16:30:26,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:30:26,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 860 16 4175 4159 19.3406 0 - 3.25676 2014-07-01 16:30:27,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 861 16 4175 4159 19.3181 0 - 3.25676 2014-07-01 16:30:28,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 862 16 4176 4160 19.3004 1.33333 2.67778 3.25662 2014-07-01 16:30:29,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 863 16 4176 4160 19.278 0 - 3.25662 2014-07-01 16:30:29,728.728 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:30:29,728.728 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:30:29,729.729 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.540728 2014-07-01 16:30:29,729.729 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 0.540728433084' 2014-07-01 16:30:29,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 1 to mira0435601-58 2014-07-01 16:30:30,421.421 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 864 16 4188 4172 19.3112 24 13.626 3.28198 2014-07-01 16:30:31,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 865 16 4189 4173 19.2935 4 0.791912 3.28138 2014-07-01 16:30:31,545.545 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 0.540728 (8354372) 2014-07-01 16:30:32,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 866 16 4189 4173 19.2712 0 - 3.28138 2014-07-01 16:30:33,423.423 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 867 16 4189 4173 19.249 0 - 3.28138 2014-07-01 16:30:34,422.422 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 868 16 4189 4173 19.2268 0 - 3.28138 2014-07-01 16:30:35,425.425 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 869 16 4189 4173 19.2047 0 - 3.28138 2014-07-01 16:30:35,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: finishing write tid 1 to mira0435601-46 2014-07-01 16:30:36,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 870 16 4189 4173 19.1826 0 - 3.28138 2014-07-01 16:30:36,555.555 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:30:36,556.556 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:30:37,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 871 16 4189 4173 19.1606 0 - 3.28138 2014-07-01 16:30:37,779.779 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:30:37,790.790 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:30:38,425.425 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 872 16 4194 4178 19.1616 2.85714 7.89476 3.28683 2014-07-01 16:30:38,800.800 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:30:38,805.805 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:30:38,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: finishing write tid 2 to mira0435601-46 2014-07-01 16:30:38,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: finishing write tid 4 to mira0435601-46 2014-07-01 16:30:39,423.423 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 873 16 4200 4184 19.1671 24 1.46803 3.29378 2014-07-01 16:30:39,794.794 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:30:39,803.803 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:30:40,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 3 to mira0435601-58 2014-07-01 16:30:40,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: finishing write tid 1 to mira0435601-322 2014-07-01 16:30:40,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: finishing write tid 3 to mira0435601-322 2014-07-01 16:30:40,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: done (6 left) 2014-07-01 16:30:40,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1310: done (5 left) 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1312: done (4 left) 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1313: done (3 left) 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 65 current snap is 0 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: seq_num 937 ranges {0=593920,593920=733184,1327104=69632} 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: writing mira0435601-65 from 0 to 593920 tid 1 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: writing mira0435601-65 from 593920 to 1327104 tid 2 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: writing mira0435601-65 from 1327104 to 1396736 tid 3 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: seq_num 938 ranges {2646016=516096,3162112=720896,3883008=495616,4378624=389120} 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: writing mira0435601-400 from 2646016 to 3162112 tid 1 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: writing mira0435601-400 from 3162112 to 3883008 tid 2 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: writing mira0435601-400 from 3883008 to 4378624 tid 3 2014-07-01 16:30:40,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: writing mira0435601-400 from 4378624 to 4767744 tid 4 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: read oid 418 snap -1 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 485 current snap is 0 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: seq_num 939 ranges {45056=544768,589824=401408} 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: writing mira0435601-485 from 45056 to 589824 tid 1 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: writing mira0435601-485 from 589824 to 991232 tid 2 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1318: read oid 209 snap -1 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: read oid 88 snap -1 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1320: read oid 244 snap -1 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 96 current snap is 0 2014-07-01 16:30:40,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: seq_num 940 ranges {5419008=413696,5832704=401408} 2014-07-01 16:30:40,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: writing mira0435601-96 from 5419008 to 5832704 tid 1 2014-07-01 16:30:40,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: writing mira0435601-96 from 5832704 to 6234112 tid 2 2014-07-01 16:30:40,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1322: read oid 457 snap -1 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 236 current snap is 0 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: seq_num 941 ranges {360448=688128} 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: writing mira0435601-236 from 360448 to 1048576 tid 1 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: read oid 158 snap -1 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: read oid 180 snap -1 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 109 current snap is 0 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: seq_num 942 ranges {1974272=716800,2691072=700416,3391488=503808} 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0435601-109 from 1974272 to 2691072 tid 1 2014-07-01 16:30:40,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0435601-109 from 2691072 to 3391488 tid 2 2014-07-01 16:30:40,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0435601-109 from 3391488 to 3895296 tid 3 2014-07-01 16:30:40,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:30:40,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 874 16 4204 4188 19.1635 16 41.8498 3.31095 2014-07-01 16:30:40,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: expect (ObjNum 560 snap 0 seq_num 560) 2014-07-01 16:30:40,817.817 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:30:40,827.827 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:30:40,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1318: expect (ObjNum 526 snap 0 seq_num 526) 2014-07-01 16:30:41,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 875 16 4216 4200 19.1964 48 2.22113 3.30626 2014-07-01 16:30:41,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: expect (ObjNum 658 snap 0 seq_num 658) 2014-07-01 16:30:41,828.828 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:30:41,838.838 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:30:41,838.838 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:41,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: expect (ObjNum 179 snap 0 seq_num 179) 2014-07-01 16:30:42,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: finishing write tid 4 to mira0435601-322 2014-07-01 16:30:42,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 183 (ObjNum 935 snap 0 seq_num 935) dirty exists 2014-07-01 16:30:42,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: finishing write tid 1 to mira0435601-485 2014-07-01 16:30:42,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: finishing write tid 2 to mira0435601-485 2014-07-01 16:30:42,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: finishing write tid 4 to mira0435601-485 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: finishing write tid 5 to mira0435601-485 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 159 (ObjNum 939 snap 0 seq_num 939) dirty exists 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: finishing write tid 1 to mira0435601-65 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: done (15 left) 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: done (14 left) 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: done (13 left) 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1318: done (12 left) 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: done (11 left) 2014-07-01 16:30:42,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: done (10 left) 2014-07-01 16:30:42,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 102 current snap is 0 2014-07-01 16:30:42,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: seq_num 943 ranges {2170880=233472} 2014-07-01 16:30:42,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: writing mira0435601-102 from 2170880 to 2404352 tid 1 2014-07-01 16:30:42,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1328: read oid 207 snap -1 2014-07-01 16:30:42,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: finishing write tid 2 to mira0435601-65 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: finishing write tid 3 to mira0435601-65 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: finishing write tid 5 to mira0435601-65 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: finishing write tid 6 to mira0435601-65 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 102 (ObjNum 937 snap 0 seq_num 937) dirty exists 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 1 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 2 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 3 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 4 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 6 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: finishing write tid 7 to mira0435601-400 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 164 (ObjNum 938 snap 0 seq_num 938) dirty exists 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: done (11 left) 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: done (10 left) 2014-07-01 16:30:42,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1329: read oid 344 snap -1 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 195 current snap is 0 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: seq_num 944 ranges {1167360=106496} 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: writing mira0435601-195 from 1167360 to 1273856 tid 1 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 499 current snap is 0 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: seq_num 945 ranges {2437120=536576,2973696=696320,3670016=106496} 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: writing mira0435601-499 from 2437120 to 2973696 tid 1 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: writing mira0435601-499 from 2973696 to 3670016 tid 2 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: writing mira0435601-499 from 3670016 to 3776512 tid 3 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1332: read oid 454 snap -1 2014-07-01 16:30:42,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: delete oid 103 current snap is 0 2014-07-01 16:30:42,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 876 16 4220 4204 19.1928 16 1.08429 3.30406 2014-07-01 16:30:42,558.558 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:42,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1329: expect (ObjNum 343 snap 0 seq_num 343) 2014-07-01 16:30:43,427.427 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 877 16 4226 4210 19.1982 24 1.97365 3.30181 2014-07-01 16:30:44,304.304 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:44,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 878 16 4226 4210 19.1764 0 - 3.30181 2014-07-01 16:30:45,055.055 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:45,424.424 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 879 16 4226 4210 19.1546 0 - 3.30181 2014-07-01 16:30:46,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:30:46.338099min lat: 0.057269 max lat: 68.3165 avg lat: 3.30181 2014-07-01 16:30:46,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:30:46,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 880 16 4226 4210 19.1328 0 - 3.30181 2014-07-01 16:30:47,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 881 16 4230 4214 19.1292 4 0.140922 3.3077 2014-07-01 16:30:48,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 882 16 4232 4216 19.1166 8 6.80739 3.3098 2014-07-01 16:30:49,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 1 to mira0435601-236 2014-07-01 16:30:49,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 3 to mira0435601-236 2014-07-01 16:30:49,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 4 to mira0435601-236 2014-07-01 16:30:49,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 104 (ObjNum 941 snap 0 seq_num 941) dirty exists 2014-07-01 16:30:49,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 883 16 4235 4219 19.1086 12 6.62322 3.31283 2014-07-01 16:30:49,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: expect (ObjNum 157 snap 0 seq_num 157) 2014-07-01 16:30:50,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 884 16 4245 4229 19.1322 40 0.207074 3.31223 2014-07-01 16:30:50,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1332: expect (ObjNum 453 snap 0 seq_num 453) 2014-07-01 16:30:51,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: finishing write tid 1 to mira0435601-195 2014-07-01 16:30:51,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: finishing write tid 3 to mira0435601-195 2014-07-01 16:30:51,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: finishing write tid 4 to mira0435601-195 2014-07-01 16:30:51,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 166 (ObjNum 944 snap 0 seq_num 944) dirty exists 2014-07-01 16:30:51,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 1 to mira0435601-102 2014-07-01 16:30:51,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 3 to mira0435601-102 2014-07-01 16:30:51,425.425 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 885 16 4248 4232 19.1241 12 0.735957 3.31096 2014-07-01 16:30:51,741.741 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:30:51,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:1328: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:30:52,309.309 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:30:39.393711', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883761680, u'health': u'HEALTH_OK', u'kb_used': 13019372, u'store_stats': {u'bytes_total': 9016956, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725484}}, {u'last_updated': u'2014-07-01 07:29:54.531317', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887059184, u'health': u'HEALTH_OK', u'kb_used': 9721868, u'store_stats': {u'bytes_total': 9016943, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725471}}, {u'last_updated': u'2014-07-01 07:30:11.314326', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887035040, u'health': u'HEALTH_OK', u'kb_used': 9746012, u'store_stats': {u'bytes_total': 10065528, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725480}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'7 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2883/9826 objects degraded (29.341%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 123541544, u'degraded_objects': 2883, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 18576885833, u'degraded_total': 9826, u'bytes_used': 31128793088, u'recovering_objects_per_sec': 32, u'write_bytes_sec': 45250238, u'op_per_sec': 30, u'version': 752, u'pgs_by_state': [{u'count': 2, u'state_name': u'inactive'}, {u'count': 7, u'state_name': u'peering'}, {u'count': 61, u'state_name': u'active+recovery_wait'}, {u'count': 150, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 43173, u'degraded_ratio': u'29.341', u'bytes_avail': 4967453745152}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 401, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:30:52,310.310 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:30:52,426.426 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 886 16 4252 4236 19.1206 16 10.0906 3.31559 2014-07-01 16:30:52,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: finishing write tid 1 to mira0435601-499 2014-07-01 16:30:52,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: finishing write tid 2 to mira0435601-499 2014-07-01 16:30:52,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: finishing write tid 3 to mira0435601-499 2014-07-01 16:30:52,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: finishing write tid 5 to mira0435601-499 2014-07-01 16:30:52,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1320: done (14 left) 2014-07-01 16:30:52,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: done (13 left) 2014-07-01 16:30:52,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: done (12 left) 2014-07-01 16:30:52,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1328: done (11 left) 2014-07-01 16:30:52,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1329: done (10 left) 2014-07-01 16:30:52,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: done (9 left) 2014-07-01 16:30:52,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1332: done (8 left) 2014-07-01 16:30:52,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: done (7 left) 2014-07-01 16:30:52,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: delete oid 304 current snap is 0 2014-07-01 16:30:52,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: finishing write tid 6 to mira0435601-499 2014-07-01 16:30:52,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 499 v 152 (ObjNum 945 snap 0 seq_num 945) dirty exists 2014-07-01 16:30:52,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1322: expect (ObjNum 892 snap 0 seq_num 892) 2014-07-01 16:30:53,102.102 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:30:53,102.102 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:53,426.426 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 887 16 4261 4245 19.1396 36 22.8428 3.31815 2014-07-01 16:30:53,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: finishing write tid 5 to mira0435601-46 2014-07-01 16:30:53,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 112 (ObjNum 932 snap 0 seq_num 932) dirty exists 2014-07-01 16:30:53,996.996 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:54,427.427 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 888 16 4262 4246 19.1226 4 3.54913 3.3182 2014-07-01 16:30:55,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 889 16 4262 4246 19.101 0 - 3.3182 2014-07-01 16:30:56,427.427 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 890 16 4262 4246 19.0796 0 - 3.3182 2014-07-01 16:30:57,427.427 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 891 16 4262 4246 19.0582 0 - 3.3182 2014-07-01 16:30:58,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 892 16 4263 4247 19.0413 1 4.55467 3.3185 2014-07-01 16:30:58,724.724 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:30:59,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 893 16 4263 4247 19.02 0 - 3.3185 2014-07-01 16:30:59,497.497 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:30:59,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 4 to mira0435601-102 2014-07-01 16:30:59,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 210 (ObjNum 943 snap 0 seq_num 943) dirty exists 2014-07-01 16:31:00,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 894 16 4271 4255 19.0345 16 7.38512 3.32698 2014-07-01 16:31:00,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: done (7 left) 2014-07-01 16:31:00,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1322: done (6 left) 2014-07-01 16:31:00,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: done (5 left) 2014-07-01 16:31:00,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: done (4 left) 2014-07-01 16:31:00,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: done (3 left) 2014-07-01 16:31:00,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 362 current snap is 0 2014-07-01 16:31:00,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: seq_num 948 ranges {1794048=753664} 2014-07-01 16:31:00,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: writing mira0435601-362 from 1794048 to 2547712 tid 1 2014-07-01 16:31:00,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 374 current snap is 0 2014-07-01 16:31:00,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: seq_num 949 ranges {172032=782336,954368=520192,1474560=405504,1880064=278528} 2014-07-01 16:31:00,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: writing mira0435601-374 from 172032 to 954368 tid 1 2014-07-01 16:31:00,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: writing mira0435601-374 from 954368 to 1474560 tid 2 2014-07-01 16:31:00,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: writing mira0435601-374 from 1474560 to 1880064 tid 3 2014-07-01 16:31:00,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: writing mira0435601-374 from 1880064 to 2158592 tid 4 2014-07-01 16:31:00,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: read oid 26 snap -1 2014-07-01 16:31:00,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:1338: delete oid 38 current snap is 0 2014-07-01 16:31:01,059.059 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:01,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 895 16 4275 4259 19.0311 16 0.571978 3.32478 2014-07-01 16:31:01,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: finishing write tid 1 to mira0435601-96 2014-07-01 16:31:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: finishing write tid 2 to mira0435601-96 2014-07-01 16:31:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: finishing write tid 4 to mira0435601-96 2014-07-01 16:31:01,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 4 to mira0435601-58 2014-07-01 16:31:01,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 90 (ObjNum 927 snap 0 seq_num 927) dirty exists 2014-07-01 16:31:01,808.808 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:30:39.393711', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883761680, u'health': u'HEALTH_OK', u'kb_used': 13019372, u'store_stats': {u'bytes_total': 9016956, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725484}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:30:11.314326', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887035040, u'health': u'HEALTH_OK', u'kb_used': 9746012, u'store_stats': {u'bytes_total': 10065528, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725480}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'66 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2909/9881 objects degraded (29.440%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5865425, u'degraded_objects': 2909, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 18695104566, u'degraded_total': 9881, u'bytes_used': 31581298688, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 9631502, u'op_per_sec': 3, u'version': 758, u'pgs_by_state': [{u'count': 66, u'state_name': u'active+recovery_wait'}, {u'count': 154, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3657, u'degraded_ratio': u'29.440', u'bytes_avail': 4967001239552}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 402, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:01,808.808 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:02,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 896 16 4293 4277 19.0902 72 0.757815 3.32679 2014-07-01 16:31:02,539.539 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:31:02,540.540 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:02,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 1 to mira0435601-362 2014-07-01 16:31:02,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 3 to mira0435601-362 2014-07-01 16:31:02,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 4 to mira0435601-362 2014-07-01 16:31:02,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 212 (ObjNum 948 snap 0 seq_num 948) dirty exists 2014-07-01 16:31:03,264.264 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:03,428.428 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 897 16 4296 4280 19.0823 12 0.437734 3.32465 2014-07-01 16:31:03,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 1 to mira0435601-109 2014-07-01 16:31:03,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 2 to mira0435601-109 2014-07-01 16:31:03,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 3 to mira0435601-109 2014-07-01 16:31:03,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 5 to mira0435601-109 2014-07-01 16:31:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: finishing write tid 5 to mira0435601-96 2014-07-01 16:31:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 194 (ObjNum 940 snap 0 seq_num 940) dirty exists 2014-07-01 16:31:03,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 6 to mira0435601-109 2014-07-01 16:31:03,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 151 (ObjNum 942 snap 0 seq_num 942) dirty exists 2014-07-01 16:31:04,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 898 16 4300 4284 19.0788 16 2.34917 3.32501 2014-07-01 16:31:05,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 1 to mira0435601-374 2014-07-01 16:31:05,429.429 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 899 16 4308 4292 19.0932 32 2.99085 3.3238 2014-07-01 16:31:05,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 2 to mira0435601-374 2014-07-01 16:31:05,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 3 to mira0435601-374 2014-07-01 16:31:05,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 4 to mira0435601-374 2014-07-01 16:31:05,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 6 to mira0435601-374 2014-07-01 16:31:05,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: finishing write tid 7 to mira0435601-374 2014-07-01 16:31:05,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 92 (ObjNum 949 snap 0 seq_num 949) dirty exists 2014-07-01 16:31:06,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:31:06.342389min lat: 0.057269 max lat: 68.3165 avg lat: 3.32144 2014-07-01 16:31:06,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:31:06,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 900 16 4318 4302 19.1164 40 0.748474 3.32144 2014-07-01 16:31:07,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 901 16 4318 4302 19.0952 0 - 3.32144 2014-07-01 16:31:07,686.686 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:08,287.287 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:08,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 902 16 4318 4302 19.074 0 - 3.32144 2014-07-01 16:31:09,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 903 16 4320 4304 19.0618 2.66667 3.17229 3.32138 2014-07-01 16:31:10,354.354 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:10,430.430 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 904 16 4320 4304 19.0407 0 - 3.32138 2014-07-01 16:31:11,147.147 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:30:39.393711', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883761680, u'health': u'HEALTH_OK', u'kb_used': 13019372, u'store_stats': {u'bytes_total': 9016956, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725484}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:30:11.314326', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887035040, u'health': u'HEALTH_OK', u'kb_used': 9746012, u'store_stats': {u'bytes_total': 10065528, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725480}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'66 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2907/9969 objects degraded (29.160%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2907, u'num_pgs': 244, u'data_bytes': 18906341392, u'degraded_total': 9969, u'bytes_used': 32213614592, u'op_per_sec': 1, u'write_bytes_sec': 5525332, u'version': 765, u'pgs_by_state': [{u'count': 66, u'state_name': u'active+recovery_wait'}, {u'count': 154, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'29.160', u'bytes_avail': 4966368923648}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 404, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:11,147.147 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:11,430.430 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 905 16 4320 4304 19.0196 0 - 3.32138 2014-07-01 16:31:11,987.987 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:31:11,987.987 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:12,430.430 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 906 16 4320 4304 18.9986 0 - 3.32138 2014-07-01 16:31:13,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: done (6 left) 2014-07-01 16:31:13,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: done (5 left) 2014-07-01 16:31:13,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: done (4 left) 2014-07-01 16:31:13,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: done (3 left) 2014-07-01 16:31:13,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: done (2 left) 2014-07-01 16:31:13,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: done (1 left) 2014-07-01 16:31:13,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1338: done (0 left) 2014-07-01 16:31:13,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: read oid 18 snap -1 2014-07-01 16:31:13,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 136 current snap is 0 2014-07-01 16:31:13,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: seq_num 951 ranges {2695168=524288,3219456=737280,3956736=466944} 2014-07-01 16:31:13,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: writing mira0435601-136 from 2695168 to 3219456 tid 1 2014-07-01 16:31:13,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: writing mira0435601-136 from 3219456 to 3956736 tid 2 2014-07-01 16:31:13,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: writing mira0435601-136 from 3956736 to 4423680 tid 3 2014-07-01 16:31:13,421.421 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 320 current snap is 0 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: seq_num 952 ranges {1306624=602112,1908736=671744,2580480=131072} 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: writing mira0435601-320 from 1306624 to 1908736 tid 1 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: writing mira0435601-320 from 1908736 to 2580480 tid 2 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: writing mira0435601-320 from 2580480 to 2711552 tid 3 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 356 current snap is 0 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: seq_num 953 ranges {1499136=720896,2220032=503808,2723840=454656,3178496=507904,3686400=139264} 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0435601-356 from 1499136 to 2220032 tid 1 2014-07-01 16:31:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0435601-356 from 2220032 to 2723840 tid 2 2014-07-01 16:31:13,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 907 16 4323 4307 18.9909 3 4.45876 3.32403 2014-07-01 16:31:13,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0435601-356 from 2723840 to 3178496 tid 3 2014-07-01 16:31:13,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0435601-356 from 3178496 to 3686400 tid 4 2014-07-01 16:31:13,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0435601-356 from 3686400 to 3825664 tid 5 2014-07-01 16:31:13,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 67 current snap is 0 2014-07-01 16:31:13,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: seq_num 954 ranges {974848=745472,1720320=401408} 2014-07-01 16:31:13,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: writing mira0435601-67 from 974848 to 1720320 tid 1 2014-07-01 16:31:13,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: writing mira0435601-67 from 1720320 to 2121728 tid 2 2014-07-01 16:31:13,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 4 current snap is 0 2014-07-01 16:31:13,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: seq_num 955 ranges {622592=774144,1396736=716800,2113536=753664,2867200=135168} 2014-07-01 16:31:13,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0435601-4 from 622592 to 1396736 tid 1 2014-07-01 16:31:13,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0435601-4 from 1396736 to 2113536 tid 2 2014-07-01 16:31:13,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0435601-4 from 2113536 to 2867200 tid 3 2014-07-01 16:31:13,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0435601-4 from 2867200 to 3002368 tid 4 2014-07-01 16:31:13,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 189 current snap is 0 2014-07-01 16:31:13,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: seq_num 956 ranges {1761280=540672,2301952=499712,2801664=626688,3428352=77824} 2014-07-01 16:31:13,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: writing mira0435601-189 from 1761280 to 2301952 tid 1 2014-07-01 16:31:13,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: writing mira0435601-189 from 2301952 to 2801664 tid 2 2014-07-01 16:31:13,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: writing mira0435601-189 from 2801664 to 3428352 tid 3 2014-07-01 16:31:13,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: writing mira0435601-189 from 3428352 to 3506176 tid 4 2014-07-01 16:31:13,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1346: read oid 366 snap -1 2014-07-01 16:31:13,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: read oid 460 snap -1 2014-07-01 16:31:13,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 383 current snap is 0 2014-07-01 16:31:13,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: seq_num 957 ranges {2392064=565248} 2014-07-01 16:31:13,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: writing mira0435601-383 from 2392064 to 2957312 tid 1 2014-07-01 16:31:13,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 260 current snap is 0 2014-07-01 16:31:13,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: seq_num 958 ranges {1736704=593920,2330624=712704,3043328=491520} 2014-07-01 16:31:13,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: writing mira0435601-260 from 1736704 to 2330624 tid 1 2014-07-01 16:31:13,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: writing mira0435601-260 from 2330624 to 3043328 tid 2 2014-07-01 16:31:13,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: writing mira0435601-260 from 3043328 to 3534848 tid 3 2014-07-01 16:31:13,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: delete oid 9 current snap is 0 2014-07-01 16:31:14,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: done (11 left) 2014-07-01 16:31:14,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: read oid 295 snap -1 2014-07-01 16:31:14,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 416 current snap is 0 2014-07-01 16:31:14,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: seq_num 960 ranges {2023424=737280,2760704=479232,3239936=323584} 2014-07-01 16:31:14,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0435601-416 from 2023424 to 2760704 tid 1 2014-07-01 16:31:14,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0435601-416 from 2760704 to 3239936 tid 2 2014-07-01 16:31:14,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0435601-416 from 3239936 to 3563520 tid 3 2014-07-01 16:31:14,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1353: delete oid 228 current snap is 0 2014-07-01 16:31:14,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 908 16 4330 4314 19.0008 28 0.234655 3.32901 2014-07-01 16:31:14,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: expect (ObjNum 915 snap 0 seq_num 915) 2014-07-01 16:31:15,431.431 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 909 16 4343 4327 19.0371 52 0.079593 3.33439 2014-07-01 16:31:16,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: expect (ObjNum 584 snap 0 seq_num 584) 2014-07-01 16:31:16,431.431 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 910 16 4347 4331 19.0338 16 0.930844 3.33231 2014-07-01 16:31:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: done (13 left) 2014-07-01 16:31:17,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1346: done (12 left) 2014-07-01 16:31:17,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: done (11 left) 2014-07-01 16:31:17,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1353: done (10 left) 2014-07-01 16:31:17,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1354: read oid 171 snap -1 2014-07-01 16:31:17,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 446 current snap is 0 2014-07-01 16:31:17,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: seq_num 962 ranges {3993600=688128,4681728=401408} 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: writing mira0435601-446 from 3993600 to 4681728 tid 1 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: writing mira0435601-446 from 4681728 to 5083136 tid 2 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 491 current snap is 0 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: seq_num 963 ranges {2273280=557056,2830336=401408} 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: writing mira0435601-491 from 2273280 to 2830336 tid 1 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: writing mira0435601-491 from 2830336 to 3231744 tid 2 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1357: read oid 254 snap -1 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 302 current snap is 0 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: seq_num 964 ranges {1957888=585728,2543616=458752,3002368=520192,3522560=598016,4120576=28672} 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: writing mira0435601-302 from 1957888 to 2543616 tid 1 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: writing mira0435601-302 from 2543616 to 3002368 tid 2 2014-07-01 16:31:17,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: writing mira0435601-302 from 3002368 to 3522560 tid 3 2014-07-01 16:31:17,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: writing mira0435601-302 from 3522560 to 4120576 tid 4 2014-07-01 16:31:17,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: writing mira0435601-302 from 4120576 to 4149248 tid 5 2014-07-01 16:31:17,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:1359: read oid 197 snap -1 2014-07-01 16:31:17,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:31:17,431.431 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 911 16 4347 4331 19.0129 0 - 3.33231 2014-07-01 16:31:18,145.145 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:18,431.431 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 912 16 4347 4331 18.9921 0 - 3.33231 2014-07-01 16:31:19,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 913 16 4347 4331 18.9713 0 - 3.33231 2014-07-01 16:31:19,866.866 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:20,432.432 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 914 16 4351 4335 18.968 4 3.75289 3.33304 2014-07-01 16:31:21,436.436 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:21,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 915 16 4351 4335 18.9473 0 - 3.33304 2014-07-01 16:31:21,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 1 to mira0435601-67 2014-07-01 16:31:21,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 2 to mira0435601-67 2014-07-01 16:31:21,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 4 to mira0435601-67 2014-07-01 16:31:22,176.176 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:30:39.393711', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883761680, u'health': u'HEALTH_OK', u'kb_used': 13019372, u'store_stats': {u'bytes_total': 9016956, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725484}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'66 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2922/10037 objects degraded (29.112%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5200009, u'degraded_objects': 2922, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 19041224701, u'degraded_total': 10037, u'bytes_used': 32583651328, u'recovering_objects_per_sec': 1, u'version': 771, u'pgs_by_state': [{u'count': 66, u'state_name': u'active+recovery_wait'}, {u'count': 155, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'29.112', u'bytes_avail': 4965998886912}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 405, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:22,177.177 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:22,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: expect (ObjNum 876 snap 0 seq_num 876) 2014-07-01 16:31:22,432.432 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 916 16 4357 4341 18.9528 12 0.210221 3.33654 2014-07-01 16:31:22,885.885 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:31:22,885.885 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:23,432.432 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 917 16 4360 4344 18.9452 12 3.29399 3.33703 2014-07-01 16:31:23,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 1 to mira0435601-189 2014-07-01 16:31:23,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 2 to mira0435601-189 2014-07-01 16:31:23,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 3 to mira0435601-189 2014-07-01 16:31:23,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 4 to mira0435601-189 2014-07-01 16:31:23,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 6 to mira0435601-189 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: finishing write tid 1 to mira0435601-260 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: finishing write tid 2 to mira0435601-260 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: finishing write tid 3 to mira0435601-260 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: finishing write tid 5 to mira0435601-260 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: done (15 left) 2014-07-01 16:31:23,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: delete oid 156 current snap is 0 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 1 to mira0435601-4 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 2 to mira0435601-4 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 3 to mira0435601-4 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 4 to mira0435601-4 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 6 to mira0435601-4 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: finishing write tid 1 to mira0435601-383 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: finishing write tid 3 to mira0435601-383 2014-07-01 16:31:23,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: finishing write tid 1 to mira0435601-446 2014-07-01 16:31:23,680.680 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:24,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: finishing write tid 2 to mira0435601-446 2014-07-01 16:31:24,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: finishing write tid 4 to mira0435601-446 2014-07-01 16:31:24,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: finishing write tid 1 to mira0435601-491 2014-07-01 16:31:24,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: finishing write tid 2 to mira0435601-491 2014-07-01 16:31:24,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: finishing write tid 4 to mira0435601-491 2014-07-01 16:31:24,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 918 16 4360 4344 18.9246 0 - 3.33703 2014-07-01 16:31:24,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 1 to mira0435601-302 2014-07-01 16:31:24,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 2 to mira0435601-302 2014-07-01 16:31:24,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 3 to mira0435601-302 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 4 to mira0435601-302 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 5 to mira0435601-302 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 7 to mira0435601-302 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: finishing write tid 8 to mira0435601-302 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 159 (ObjNum 964 snap 0 seq_num 964) dirty exists 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: finishing write tid 5 to mira0435601-491 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 218 (ObjNum 963 snap 0 seq_num 963) dirty exists 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: finishing write tid 5 to mira0435601-446 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 215 (ObjNum 962 snap 0 seq_num 962) dirty exists 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: finishing write tid 1 to mira0435601-136 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: finishing write tid 2 to mira0435601-136 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: finishing write tid 3 to mira0435601-136 2014-07-01 16:31:24,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: finishing write tid 5 to mira0435601-136 2014-07-01 16:31:24,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: finishing write tid 6 to mira0435601-136 2014-07-01 16:31:24,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 212 (ObjNum 951 snap 0 seq_num 951) dirty exists 2014-07-01 16:31:25,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 919 16 4363 4347 18.917 6 1.51556 3.33684 2014-07-01 16:31:26,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:31:26.346472min lat: 0.057269 max lat: 68.3165 avg lat: 3.33532 2014-07-01 16:31:26,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:31:26,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 920 16 4366 4350 18.9095 12 0.17511 3.33532 2014-07-01 16:31:26,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: finishing write tid 4 to mira0435601-383 2014-07-01 16:31:26,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 80 (ObjNum 957 snap 0 seq_num 957) dirty exists 2014-07-01 16:31:26,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 7 to mira0435601-4 2014-07-01 16:31:26,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 169 (ObjNum 955 snap 0 seq_num 955) dirty exists 2014-07-01 16:31:26,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: finishing write tid 6 to mira0435601-260 2014-07-01 16:31:26,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 163 (ObjNum 958 snap 0 seq_num 958) dirty exists 2014-07-01 16:31:26,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: finishing write tid 7 to mira0435601-189 2014-07-01 16:31:26,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 159 (ObjNum 956 snap 0 seq_num 956) dirty exists 2014-07-01 16:31:26,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 5 to mira0435601-67 2014-07-01 16:31:26,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 156 (ObjNum 954 snap 0 seq_num 954) dirty exists 2014-07-01 16:31:26,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:1357: expect (ObjNum 253 snap 0 seq_num 253) 2014-07-01 16:31:27,434.434 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 921 16 4381 4365 18.9541 60 0.307828 3.34429 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: done (15 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: done (14 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: done (13 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: done (12 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: done (11 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: done (10 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: done (9 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: done (8 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1357: done (7 left) 2014-07-01 16:31:27,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: done (6 left) 2014-07-01 16:31:27,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: done (5 left) 2014-07-01 16:31:27,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 424 current snap is 0 2014-07-01 16:31:27,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: seq_num 966 ranges {2588672=376832} 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: writing mira0435601-424 from 2588672 to 2965504 tid 1 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 165 current snap is 0 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: seq_num 967 ranges {0=405504,405504=749568,1155072=446464,1601536=8192} 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: writing mira0435601-165 from 0 to 405504 tid 1 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: writing mira0435601-165 from 405504 to 1155072 tid 2 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: writing mira0435601-165 from 1155072 to 1601536 tid 3 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: writing mira0435601-165 from 1601536 to 1609728 tid 4 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: read oid 463 snap -1 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 100 current snap is 0 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: seq_num 968 ranges {2162688=675840,2838528=684032,3522560=118784} 2014-07-01 16:31:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: writing mira0435601-100 from 2162688 to 2838528 tid 1 2014-07-01 16:31:27,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: writing mira0435601-100 from 2838528 to 3522560 tid 2 2014-07-01 16:31:27,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: writing mira0435601-100 from 3522560 to 3641344 tid 3 2014-07-01 16:31:27,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1365: read oid 261 snap -1 2014-07-01 16:31:27,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: expect (ObjNum 462 snap 0 seq_num 462) 2014-07-01 16:31:28,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1354: expect (ObjNum 170 snap 0 seq_num 170) 2014-07-01 16:31:28,193.193 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:28,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1359: expect (ObjNum 744 snap 0 seq_num 744) 2014-07-01 16:31:28,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 922 16 4392 4376 18.9813 44 0.275351 3.34001 2014-07-01 16:31:28,784.784 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:29,433.433 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 923 16 4392 4376 18.9607 0 - 3.34001 2014-07-01 16:31:30,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 1 to mira0435601-356 2014-07-01 16:31:30,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 2 to mira0435601-356 2014-07-01 16:31:30,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 3 to mira0435601-356 2014-07-01 16:31:30,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 4 to mira0435601-356 2014-07-01 16:31:30,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 5 to mira0435601-356 2014-07-01 16:31:30,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 7 to mira0435601-356 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: finishing write tid 1 to mira0435601-100 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: finishing write tid 1 to mira0435601-424 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: finishing write tid 3 to mira0435601-424 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: finishing write tid 2 to mira0435601-100 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1354: done (9 left) 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1359: done (8 left) 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: done (7 left) 2014-07-01 16:31:30,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: read oid 434 snap -1 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: finishing write tid 3 to mira0435601-100 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: finishing write tid 5 to mira0435601-100 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 1 to mira0435601-165 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: finishing write tid 6 to mira0435601-100 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 170 (ObjNum 968 snap 0 seq_num 968) dirty exists 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 2 to mira0435601-165 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: finishing write tid 4 to mira0435601-424 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 161 (ObjNum 966 snap 0 seq_num 966) dirty exists 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 3 to mira0435601-165 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 4 to mira0435601-165 2014-07-01 16:31:30,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: done (7 left) 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: done (6 left) 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1367: delete oid 427 current snap is 0 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 6 to mira0435601-165 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: finishing write tid 7 to mira0435601-165 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 166 (ObjNum 967 snap 0 seq_num 967) dirty exists 2014-07-01 16:31:30,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: expect (ObjNum 928 snap 0 seq_num 928) 2014-07-01 16:31:30,434.434 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 924 16 4392 4376 18.9402 0 - 3.34001 2014-07-01 16:31:30,514.514 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:31,222.222 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:30:39.393711', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883761680, u'health': u'HEALTH_OK', u'kb_used': 13019372, u'store_stats': {u'bytes_total': 9016956, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 2725484}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2922/10120 objects degraded (28.874%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2922, u'num_pgs': 244, u'data_bytes': 19224186801, u'degraded_total': 10120, u'bytes_used': 32756588544, u'version': 780, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'28.874', u'bytes_avail': 4965825949696}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 409, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:31,222.222 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:31,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 925 16 4392 4376 18.9197 0 - 3.34001 2014-07-01 16:31:31,995.995 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:31:31,996.996 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:32,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 926 16 4392 4376 18.8993 0 - 3.34001 2014-07-01 16:31:32,899.899 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:33,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 8 to mira0435601-356 2014-07-01 16:31:33,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 356 v 96 (ObjNum 953 snap 0 seq_num 953) dirty exists 2014-07-01 16:31:33,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1365: expect (ObjNum 810 snap 0 seq_num 810) 2014-07-01 16:31:33,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 927 16 4404 4388 18.9307 9.6 5.28345 3.34805 2014-07-01 16:31:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 1 to mira0435601-416 2014-07-01 16:31:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 2 to mira0435601-416 2014-07-01 16:31:33,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 3 to mira0435601-416 2014-07-01 16:31:33,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 5 to mira0435601-416 2014-07-01 16:31:34,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 928 16 4406 4390 18.9189 8 0.313994 3.34783 2014-07-01 16:31:35,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 929 16 4406 4390 18.8985 0 - 3.34783 2014-07-01 16:31:35,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 6 to mira0435601-416 2014-07-01 16:31:35,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 185 (ObjNum 960 snap 0 seq_num 960) dirty exists 2014-07-01 16:31:35,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: finishing write tid 1 to mira0435601-320 2014-07-01 16:31:35,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: finishing write tid 2 to mira0435601-320 2014-07-01 16:31:35,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: finishing write tid 3 to mira0435601-320 2014-07-01 16:31:35,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: finishing write tid 5 to mira0435601-320 2014-07-01 16:31:36,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: finishing write tid 6 to mira0435601-320 2014-07-01 16:31:36,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 155 (ObjNum 952 snap 0 seq_num 952) dirty exists 2014-07-01 16:31:36,435.435 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 930 16 4429 4413 18.9771 46 0.433904 3.34135 2014-07-01 16:31:37,436.436 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:37,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 931 16 4429 4413 18.9567 0 - 3.34135 2014-07-01 16:31:38,244.244 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:38,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 932 16 4430 4414 18.9407 2 1.69278 3.34098 2014-07-01 16:31:39,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 933 16 4434 4418 18.9375 16 2.5736 3.34027 2014-07-01 16:31:40,089.089 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:40,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 934 16 4434 4418 18.9172 0 - 3.34027 2014-07-01 16:31:40,890.890 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2942/10193 objects degraded (28.863%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 4144898, u'degraded_objects': 2942, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 19379304274, u'degraded_total': 10193, u'bytes_used': 33158750208, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 18656092, u'op_per_sec': 10, u'version': 789, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 14673, u'degraded_ratio': u'28.863', u'bytes_avail': 4965423788032}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 413, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:40,890.890 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:41,436.436 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 935 16 4435 4419 18.9013 2 2.71556 3.34013 2014-07-01 16:31:41,531.531 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:31:41,531.531 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:42,320.320 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:42,437.437 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 936 16 4440 4424 18.9024 20 6.75478 3.34222 2014-07-01 16:31:43,437.437 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 937 16 4446 4430 18.9079 24 0.091949 3.34016 2014-07-01 16:31:44,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 938 16 4446 4430 18.8877 0 - 3.34016 2014-07-01 16:31:45,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 939 16 4446 4430 18.8676 0 - 3.34016 2014-07-01 16:31:46,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:31:46.350381min lat: 0.057269 max lat: 68.3165 avg lat: 3.33886 2014-07-01 16:31:46,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:31:46,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 940 16 4450 4434 18.8645 5.33333 0.194284 3.33886 2014-07-01 16:31:46,850.850 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:47,437.437 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 941 16 4462 4446 18.8955 48 0.23608 3.34694 2014-07-01 16:31:47,730.730 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:48,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 942 16 4468 4452 18.9009 24 0.305057 3.34289 2014-07-01 16:31:49,284.284 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:49,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 943 16 4469 4453 18.8851 4 1.37879 3.34244 2014-07-01 16:31:50,018.018 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:30:54.531709', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887010440, u'health': u'HEALTH_OK', u'kb_used': 9770612, u'store_stats': {u'bytes_total': 3127361, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2968/10277 objects degraded (28.880%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12472839, u'degraded_objects': 2968, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 19563275014, u'degraded_total': 10277, u'bytes_used': 33615933440, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 54058105, u'op_per_sec': 33, u'version': 799, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 43646, u'degraded_ratio': u'28.880', u'bytes_avail': 4964966604800}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 418, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:50,018.018 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:50,438.438 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 944 16 4473 4457 18.882 16 0.07328 3.3404 2014-07-01 16:31:50,776.776 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:31:50,776.776 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:51,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 945 16 4474 4458 18.8663 4 1.0257 3.33988 2014-07-01 16:31:51,545.545 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:52,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 946 16 4474 4458 18.8464 0 - 3.33988 2014-07-01 16:31:53,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 947 16 4476 4460 18.8349 4 17.1105 3.34292 2014-07-01 16:31:54,439.439 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 948 16 4478 4462 18.8235 8 14.7101 3.34802 2014-07-01 16:31:55,441.441 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 949 16 4478 4462 18.8036 0 - 3.34802 2014-07-01 16:31:56,054.054 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:56,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: done (6 left) 2014-07-01 16:31:56,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: done (5 left) 2014-07-01 16:31:56,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: done (4 left) 2014-07-01 16:31:56,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: done (3 left) 2014-07-01 16:31:56,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1365: done (2 left) 2014-07-01 16:31:56,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: done (1 left) 2014-07-01 16:31:56,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1367: done (0 left) 2014-07-01 16:31:56,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1368: read oid 242 snap -1 2014-07-01 16:31:56,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 261 current snap is 0 2014-07-01 16:31:56,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: seq_num 970 ranges {655360=471040,1126400=499712,1626112=466944,2093056=638976} 2014-07-01 16:31:56,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 950 16 4493 4477 18.847 30 3.72481 3.35866 2014-07-01 16:31:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: writing mira0435601-261 from 655360 to 1126400 tid 1 2014-07-01 16:31:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: writing mira0435601-261 from 1126400 to 1626112 tid 2 2014-07-01 16:31:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: writing mira0435601-261 from 1626112 to 2093056 tid 3 2014-07-01 16:31:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: writing mira0435601-261 from 2093056 to 2732032 tid 4 2014-07-01 16:31:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1370: read oid 254 snap -1 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: seq_num 971 ranges {2949120=495616,3444736=401408} 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: writing mira0435601-27 from 2949120 to 3444736 tid 1 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: writing mira0435601-27 from 3444736 to 3846144 tid 2 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: read oid 24 snap -1 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: read oid 362 snap -1 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: delete oid 9 current snap is 0 2014-07-01 16:31:56,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1368: expect (ObjNum 742 snap 0 seq_num 742) 2014-07-01 16:31:56,590.590 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:31:57,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 951 16 4508 4492 18.8902 60 0.2172 3.35161 2014-07-01 16:31:58,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: expect (ObjNum 23 snap 0 seq_num 23) 2014-07-01 16:31:58,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: expect (ObjNum 948 snap 0 seq_num 948) 2014-07-01 16:31:58,321.321 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:58,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 952 16 4510 4494 18.8788 8 0.192945 3.35019 2014-07-01 16:31:59,045.045 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'10 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2979/10313 objects degraded (28.886%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2979, u'num_pgs': 244, u'data_bytes': 19640849082, u'degraded_total': 10313, u'bytes_used': 33904230400, u'op_per_sec': 11, u'write_bytes_sec': 14523379, u'version': 811, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3195427, u'degraded_ratio': u'28.886', u'bytes_avail': 4964678307840}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 424, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:31:59,046.046 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:31:59,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 953 16 4511 4495 18.8632 4 1.27977 3.34973 2014-07-01 16:31:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1368: done (6 left) 2014-07-01 16:31:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: done (5 left) 2014-07-01 16:31:59,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: done (4 left) 2014-07-01 16:31:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: done (3 left) 2014-07-01 16:31:59,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 110 current snap is 0 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: seq_num 973 ranges {286720=643072} 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: writing mira0435601-110 from 286720 to 929792 tid 1 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1376: read oid 244 snap -1 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 181 current snap is 0 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: seq_num 974 ranges {1822720=315392} 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: writing mira0435601-181 from 1822720 to 2138112 tid 1 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 45 current snap is 0 2014-07-01 16:31:59,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: seq_num 975 ranges {3215360=745472,3960832=712704,4673536=90112} 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: writing mira0435601-45 from 3215360 to 3960832 tid 1 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: writing mira0435601-45 from 3960832 to 4673536 tid 2 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: writing mira0435601-45 from 4673536 to 4763648 tid 3 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 212 current snap is 0 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: seq_num 976 ranges {1904640=61440} 2014-07-01 16:31:59,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: writing mira0435601-212 from 1904640 to 1966080 tid 1 2014-07-01 16:31:59,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 72 current snap is 0 2014-07-01 16:31:59,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: seq_num 977 ranges {1499136=487424,1986560=475136,2461696=327680} 2014-07-01 16:31:59,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: writing mira0435601-72 from 1499136 to 1986560 tid 1 2014-07-01 16:31:59,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: writing mira0435601-72 from 1986560 to 2461696 tid 2 2014-07-01 16:31:59,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: writing mira0435601-72 from 2461696 to 2789376 tid 3 2014-07-01 16:31:59,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 440 current snap is 0 2014-07-01 16:31:59,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: seq_num 978 ranges {987136=114688} 2014-07-01 16:31:59,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: writing mira0435601-440 from 987136 to 1101824 tid 1 2014-07-01 16:31:59,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 408 current snap is 0 2014-07-01 16:31:59,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: seq_num 979 ranges {2351104=684032,3035136=536576,3571712=634880,4206592=32768} 2014-07-01 16:31:59,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: writing mira0435601-408 from 2351104 to 3035136 tid 1 2014-07-01 16:31:59,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: writing mira0435601-408 from 3035136 to 3571712 tid 2 2014-07-01 16:31:59,829.829 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:31:59,829.829 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: writing mira0435601-408 from 3571712 to 4206592 tid 3 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: writing mira0435601-408 from 4206592 to 4239360 tid 4 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: read oid 473 snap -1 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: read oid 285 snap -1 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: read oid 412 snap -1 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 234 current snap is 0 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: seq_num 980 ranges {3547136=708608} 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: writing mira0435601-234 from 3547136 to 4255744 tid 1 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1387: read oid 258 snap -1 2014-07-01 16:31:59,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:32:00,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 954 16 4511 4495 18.8434 0 - 3.34973 2014-07-01 16:32:00,604.604 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:01,440.440 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 955 16 4511 4495 18.8237 0 - 3.34973 2014-07-01 16:32:02,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 956 16 4511 4495 18.804 0 - 3.34973 2014-07-01 16:32:03,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 957 16 4512 4496 18.7885 1 5.59553 3.35023 2014-07-01 16:32:04,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1376: done (15 left) 2014-07-01 16:32:04,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1388: read oid 84 snap -1 2014-07-01 16:32:04,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:32:04,446.446 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 958 16 4512 4496 18.7689 0 - 3.35023 2014-07-01 16:32:04,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 1 to mira0435601-181 2014-07-01 16:32:04,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 3 to mira0435601-181 2014-07-01 16:32:04,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 1 to mira0435601-234 2014-07-01 16:32:04,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 3 to mira0435601-234 2014-07-01 16:32:04,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 1 to mira0435601-261 2014-07-01 16:32:04,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1387: expect (ObjNum 788 snap 0 seq_num 788) 2014-07-01 16:32:05,037.037 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:05,441.441 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 959 16 4526 4510 18.8077 28 7.72246 3.36017 2014-07-01 16:32:06,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:32:06.354431min lat: 0.057269 max lat: 68.3165 avg lat: 3.35711 2014-07-01 16:32:06,443.443 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:32:06,443.443 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 960 16 4531 4515 18.809 20 0.635168 3.35711 2014-07-01 16:32:06,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:1388: expect (ObjNum 618 snap 0 seq_num 618) 2014-07-01 16:32:07,441.441 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 961 16 4531 4515 18.7894 0 - 3.35711 2014-07-01 16:32:07,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 1 to mira0435601-408 2014-07-01 16:32:07,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 2 to mira0435601-408 2014-07-01 16:32:07,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 3 to mira0435601-408 2014-07-01 16:32:07,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 4 to mira0435601-408 2014-07-01 16:32:07,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 6 to mira0435601-408 2014-07-01 16:32:07,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: finishing write tid 7 to mira0435601-408 2014-07-01 16:32:07,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 175 (ObjNum 979 snap 0 seq_num 979) dirty exists 2014-07-01 16:32:07,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: expect (ObjNum 819 snap 0 seq_num 819) 2014-07-01 16:32:08,441.441 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 962 16 4538 4522 18.799 14 3.02637 3.3596 2014-07-01 16:32:08,904.904 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:09,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 963 16 4545 4529 18.8085 28 0.153836 3.35754 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: finishing write tid 1 to mira0435601-72 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: finishing write tid 2 to mira0435601-72 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: finishing write tid 3 to mira0435601-72 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: finishing write tid 5 to mira0435601-72 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: finishing write tid 6 to mira0435601-72 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 170 (ObjNum 977 snap 0 seq_num 977) dirty exists 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 4 to mira0435601-181 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 259 (ObjNum 974 snap 0 seq_num 974) dirty exists 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 4 to mira0435601-234 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 261 (ObjNum 980 snap 0 seq_num 980) dirty exists 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 2 to mira0435601-261 2014-07-01 16:32:10,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 3 to mira0435601-261 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 4 to mira0435601-261 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 6 to mira0435601-261 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: finishing write tid 1 to mira0435601-27 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: finishing write tid 2 to mira0435601-27 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: finishing write tid 4 to mira0435601-27 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: finishing write tid 5 to mira0435601-27 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 172 (ObjNum 971 snap 0 seq_num 971) dirty exists 2014-07-01 16:32:10,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1370: expect (ObjNum 253 snap 0 seq_num 253) 2014-07-01 16:32:10,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 964 16 4546 4530 18.7932 4 0.266848 3.35686 2014-07-01 16:32:10,510.510 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:10,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: finishing write tid 7 to mira0435601-261 2014-07-01 16:32:10,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 171 (ObjNum 970 snap 0 seq_num 970) dirty exists 2014-07-01 16:32:10,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: expect (ObjNum 731 snap 0 seq_num 731) 2014-07-01 16:32:11,223.223 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:31:11.314755', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 887001376, u'health': u'HEALTH_OK', u'kb_used': 9779676, u'store_stats': {u'bytes_total': 5158992, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2993/10418 objects degraded (28.729%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 34606713, u'degraded_objects': 2993, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 19869565531, u'degraded_total': 10418, u'bytes_used': 34399219712, u'recovering_objects_per_sec': 9, u'write_bytes_sec': 46947766, u'op_per_sec': 32, u'version': 821, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3665108, u'degraded_ratio': u'28.729', u'bytes_avail': 4964183318528}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 426, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:32:11,223.223 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:11,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 965 16 4546 4530 18.7737 0 - 3.35686 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: finishing write tid 1 to mira0435601-440 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: finishing write tid 3 to mira0435601-440 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: done (15 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1370: done (14 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: done (13 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: done (12 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: done (11 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: done (10 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: done (9 left) 2014-07-01 16:32:11,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: done (8 left) 2014-07-01 16:32:11,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: done (7 left) 2014-07-01 16:32:11,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1387: done (6 left) 2014-07-01 16:32:11,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1388: done (5 left) 2014-07-01 16:32:11,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1389: read oid 182 snap -1 2014-07-01 16:32:11,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1390: delete oid 327 current snap is 0 2014-07-01 16:32:11,995.995 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:32:11,995.995 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:12,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 966 16 4546 4530 18.7542 0 - 3.35686 2014-07-01 16:32:12,772.772 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:13,442.442 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 967 16 4546 4530 18.7348 0 - 3.35686 2014-07-01 16:32:14,443.443 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 968 16 4546 4530 18.7155 0 - 3.35686 2014-07-01 16:32:15,443.443 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 969 16 4547 4531 18.7003 0.8 5.39747 3.35731 2014-07-01 16:32:16,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 970 16 4549 4533 18.6893 8 0.092569 3.35619 2014-07-01 16:32:16,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: finishing write tid 1 to mira0435601-212 2014-07-01 16:32:16,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: finishing write tid 3 to mira0435601-212 2014-07-01 16:32:17,173.173 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:17,443.443 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 971 16 4560 4544 18.7153 44 0.410877 3.36924 2014-07-01 16:32:17,897.897 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:18,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: expect (ObjNum 284 snap 0 seq_num 284) 2014-07-01 16:32:18,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: finishing write tid 4 to mira0435601-212 2014-07-01 16:32:18,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 174 (ObjNum 976 snap 0 seq_num 976) dirty exists 2014-07-01 16:32:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 1 to mira0435601-110 2014-07-01 16:32:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 3 to mira0435601-110 2014-07-01 16:32:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 4 to mira0435601-110 2014-07-01 16:32:18,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 100 (ObjNum 973 snap 0 seq_num 973) dirty exists 2014-07-01 16:32:18,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 972 16 4566 4550 18.7208 24 8.6915 3.37529 2014-07-01 16:32:19,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: finishing write tid 4 to mira0435601-440 2014-07-01 16:32:19,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 98 (ObjNum 978 snap 0 seq_num 978) dirty exists 2014-07-01 16:32:19,453.453 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:19,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 973 16 4575 4559 18.7385 36 2.69365 3.37404 2014-07-01 16:32:20,173.173 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:32:11.315194', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886953144, u'health': u'HEALTH_OK', u'kb_used': 9827908, u'store_stats': {u'bytes_total': 10401872, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 3006/10458 objects degraded (28.744%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 3006, u'num_pgs': 244, u'data_bytes': 19954410056, u'degraded_total': 10458, u'bytes_used': 34825801728, u'op_per_sec': 0, u'write_bytes_sec': 4119231, u'version': 831, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'28.744', u'bytes_avail': 4963756736512}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 429, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:32:20,173.173 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:20,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 974 16 4576 4560 18.7234 4 0.322204 3.37337 2014-07-01 16:32:21,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 975 16 4576 4560 18.7042 0 - 3.37337 2014-07-01 16:32:22,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 976 16 4576 4560 18.685 0 - 3.37337 2014-07-01 16:32:23,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 977 16 4576 4560 18.6659 0 - 3.37337 2014-07-01 16:32:23,951.951 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:32:23,951.951 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:24,444.444 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 978 16 4576 4560 18.6468 0 - 3.37337 2014-07-01 16:32:24,815.815 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:25,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 979 16 4584 4568 18.6604 6.4 0.332742 3.37214 2014-07-01 16:32:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: done (6 left) 2014-07-01 16:32:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: done (5 left) 2014-07-01 16:32:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: done (4 left) 2014-07-01 16:32:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: done (3 left) 2014-07-01 16:32:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1389: done (2 left) 2014-07-01 16:32:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:1390: done (1 left) 2014-07-01 16:32:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 203 current snap is 0 2014-07-01 16:32:26,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: seq_num 982 ranges {2072576=454656} 2014-07-01 16:32:26,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: writing mira0435601-203 from 2072576 to 2527232 tid 1 2014-07-01 16:32:26,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1392: read oid 328 snap -1 2014-07-01 16:32:26,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: delete oid 155 current snap is 0 2014-07-01 16:32:26,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: finishing write tid 1 to mira0435601-203 2014-07-01 16:32:26,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: finishing write tid 3 to mira0435601-203 2014-07-01 16:32:26,446.446 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:32:26.358290min lat: 0.057269 max lat: 68.3165 avg lat: 3.3768 2014-07-01 16:32:26,446.446 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:32:26,446.446 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 980 16 4596 4580 18.6904 48 0.226913 3.3768 2014-07-01 16:32:26,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:1392: expect (ObjNum 604 snap 0 seq_num 604) 2014-07-01 16:32:27,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 981 16 4603 4587 18.6998 28 0.586474 3.37744 2014-07-01 16:32:28,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 982 16 4603 4587 18.6808 0 - 3.37744 2014-07-01 16:32:29,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 983 16 4603 4587 18.6618 0 - 3.37744 2014-07-01 16:32:29,784.784 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:30,357.357 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:30,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 984 16 4603 4587 18.6428 0 - 3.37744 2014-07-01 16:32:30,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: finishing write tid 4 to mira0435601-203 2014-07-01 16:32:30,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 212 (ObjNum 982 snap 0 seq_num 982) dirty exists 2014-07-01 16:32:30,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: done (3 left) 2014-07-01 16:32:30,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1392: done (2 left) 2014-07-01 16:32:30,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: done (1 left) 2014-07-01 16:32:30,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1394: read oid 290 snap -1 2014-07-01 16:32:30,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 98 current snap is 0 2014-07-01 16:32:30,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: seq_num 984 ranges {1261568=557056,1818624=454656,2273280=348160} 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: writing mira0435601-98 from 1261568 to 1818624 tid 1 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: writing mira0435601-98 from 1818624 to 2273280 tid 2 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: writing mira0435601-98 from 2273280 to 2621440 tid 3 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 217 current snap is 0 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: seq_num 985 ranges {2396160=430080,2826240=684032,3510272=118784} 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: writing mira0435601-217 from 2396160 to 2826240 tid 1 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: writing mira0435601-217 from 2826240 to 3510272 tid 2 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: writing mira0435601-217 from 3510272 to 3629056 tid 3 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 302 current snap is 0 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: seq_num 986 ranges {4149248=700416,4849664=401408} 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: writing mira0435601-302 from 4149248 to 4849664 tid 1 2014-07-01 16:32:31,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: writing mira0435601-302 from 4849664 to 5251072 tid 2 2014-07-01 16:32:31,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 322 current snap is 0 2014-07-01 16:32:31,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: seq_num 987 ranges {2195456=729088,2924544=499712,3424256=528384,3952640=577536} 2014-07-01 16:32:31,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0435601-322 from 2195456 to 2924544 tid 1 2014-07-01 16:32:31,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0435601-322 from 2924544 to 3424256 tid 2 2014-07-01 16:32:31,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0435601-322 from 3424256 to 3952640 tid 3 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0435601-322 from 3952640 to 4530176 tid 4 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: read oid 244 snap -1 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 432 current snap is 0 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: seq_num 988 ranges {2756608=651264} 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: writing mira0435601-432 from 2756608 to 3407872 tid 1 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1401: read oid 247 snap -1 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 262 current snap is 0 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: seq_num 989 ranges {1478656=520192} 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: writing mira0435601-262 from 1478656 to 1998848 tid 1 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: seq_num 990 ranges {3563520=548864,4112384=491520,4603904=700416,5304320=12288} 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: writing mira0435601-435 from 3563520 to 4112384 tid 1 2014-07-01 16:32:31,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: writing mira0435601-435 from 4112384 to 4603904 tid 2 2014-07-01 16:32:31,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: writing mira0435601-435 from 4603904 to 5304320 tid 3 2014-07-01 16:32:31,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: writing mira0435601-435 from 5304320 to 5316608 tid 4 2014-07-01 16:32:31,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 113 current snap is 0 2014-07-01 16:32:31,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: seq_num 991 ranges {1261568=417792,1679360=479232,2158592=577536,2736128=147456} 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: writing mira0435601-113 from 1261568 to 1679360 tid 1 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: writing mira0435601-113 from 1679360 to 2158592 tid 2 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: writing mira0435601-113 from 2158592 to 2736128 tid 3 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: writing mira0435601-113 from 2736128 to 2883584 tid 4 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: read oid 257 snap -1 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1406: read oid 452 snap -1 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 3 current snap is 0 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: seq_num 992 ranges {3244032=741376,3985408=659456,4644864=647168,5292032=299008} 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: writing mira0435601-3 from 3244032 to 3985408 tid 1 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: writing mira0435601-3 from 3985408 to 4644864 tid 2 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: writing mira0435601-3 from 4644864 to 5292032 tid 3 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: writing mira0435601-3 from 5292032 to 5591040 tid 4 2014-07-01 16:32:31,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1408: delete oid 412 current snap is 0 2014-07-01 16:32:31,445.445 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 985 16 4616 4600 18.6767 13 0.550601 3.38246 2014-07-01 16:32:32,106.106 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:32,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 1 to mira0435601-435 2014-07-01 16:32:32,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 2 to mira0435601-435 2014-07-01 16:32:32,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 3 to mira0435601-435 2014-07-01 16:32:32,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 4 to mira0435601-435 2014-07-01 16:32:32,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 6 to mira0435601-435 2014-07-01 16:32:32,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: finishing write tid 1 to mira0435601-98 2014-07-01 16:32:32,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: finishing write tid 7 to mira0435601-435 2014-07-01 16:32:32,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 266 (ObjNum 990 snap 0 seq_num 990) dirty exists 2014-07-01 16:32:32,446.446 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 986 16 4619 4603 18.6699 12 1.28024 3.38112 2014-07-01 16:32:33,447.447 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 987 16 4620 4604 18.6551 4 5.67885 3.38161 2014-07-01 16:32:34,447.447 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 988 16 4621 4605 18.6402 4 2.00411 3.38132 2014-07-01 16:32:35,447.447 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 989 16 4621 4605 18.6214 0 - 3.38132 2014-07-01 16:32:35,811.811 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:31:39.394036', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883688472, u'health': u'HEALTH_OK', u'kb_used': 13092580, u'store_stats': {u'bytes_total': 6207889, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:32:11.315194', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886953144, u'health': u'HEALTH_OK', u'kb_used': 9827908, u'store_stats': {u'bytes_total': 10401872, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'37 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 3026/10562 objects degraded (28.650%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 20453953, u'degraded_objects': 3026, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20181873091, u'degraded_total': 10562, u'bytes_used': 35307843584, u'recovering_objects_per_sec': 4, u'write_bytes_sec': 44998696, u'op_per_sec': 11, u'version': 845, u'pgs_by_state': [{u'count': 65, u'state_name': u'active+recovery_wait'}, {u'count': 157, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'28.650', u'bytes_avail': 4963274694656}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 435, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:32:35,812.812 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:36,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 1 to mira0435601-113 2014-07-01 16:32:36,447.447 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 990 16 4621 4605 18.6026 0 - 3.38132 2014-07-01 16:32:36,563.563 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:32:36,563.563 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:37,445.445 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:37,464.464 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 991 16 4621 4605 18.5838 0 - 3.38132 2014-07-01 16:32:38,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 2 to mira0435601-113 2014-07-01 16:32:38,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 3 to mira0435601-113 2014-07-01 16:32:38,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 4 to mira0435601-113 2014-07-01 16:32:38,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 6 to mira0435601-113 2014-07-01 16:32:38,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 992 16 4621 4605 18.5651 0 - 3.38132 2014-07-01 16:32:38,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: finishing write tid 7 to mira0435601-113 2014-07-01 16:32:38,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 113 v 217 (ObjNum 991 snap 0 seq_num 991) dirty exists 2014-07-01 16:32:39,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: finishing write tid 1 to mira0435601-302 2014-07-01 16:32:39,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: finishing write tid 2 to mira0435601-302 2014-07-01 16:32:39,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: finishing write tid 4 to mira0435601-302 2014-07-01 16:32:39,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: finishing write tid 5 to mira0435601-302 2014-07-01 16:32:39,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 173 (ObjNum 986 snap 0 seq_num 986) dirty exists 2014-07-01 16:32:39,448.448 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 993 16 4622 4606 18.5504 0.8 7.08398 3.38212 2014-07-01 16:32:39,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: done (15 left) 2014-07-01 16:32:39,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: done (14 left) 2014-07-01 16:32:39,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: done (13 left) 2014-07-01 16:32:39,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: done (12 left) 2014-07-01 16:32:39,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1406: done (11 left) 2014-07-01 16:32:39,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1408: done (10 left) 2014-07-01 16:32:39,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 321 current snap is 0 2014-07-01 16:32:39,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: seq_num 994 ranges {2228224=638976,2867200=401408} 2014-07-01 16:32:39,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: writing mira0435601-321 from 2228224 to 2867200 tid 1 2014-07-01 16:32:39,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: writing mira0435601-321 from 2867200 to 3268608 tid 2 2014-07-01 16:32:39,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 333 current snap is 0 2014-07-01 16:32:39,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: seq_num 995 ranges {409600=512000,921600=401408} 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: writing mira0435601-333 from 409600 to 921600 tid 1 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: writing mira0435601-333 from 921600 to 1323008 tid 2 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1411: read oid 427 snap -1 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 1 to mira0435601-321 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 2 to mira0435601-321 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 4 to mira0435601-321 2014-07-01 16:32:39,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 5 to mira0435601-321 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 146 (ObjNum 994 snap 0 seq_num 994) dirty exists 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: done (12 left) 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1412: read oid 341 snap -1 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: read oid 83 snap -1 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: read oid 337 snap -1 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1415: read oid 441 snap -1 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1415: done (15 left) 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 117 current snap is 0 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: seq_num 996 ranges {0=540672,540672=638976,1179648=696320,1875968=270336} 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: writing mira0435601-117 from 0 to 540672 tid 1 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: writing mira0435601-117 from 540672 to 1179648 tid 2 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: writing mira0435601-117 from 1179648 to 1875968 tid 3 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: writing mira0435601-117 from 1875968 to 2146304 tid 4 2014-07-01 16:32:39,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:32:39,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: expect (ObjNum 906 snap 0 seq_num 906) 2014-07-01 16:32:40,447.447 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 994 16 4622 4606 18.5317 0 - 3.38212 2014-07-01 16:32:41,448.448 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 995 16 4637 4621 18.5734 30 0.215387 3.39721 2014-07-01 16:32:41,976.976 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:42,448.448 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 996 16 4644 4628 18.5829 28 1.15562 3.39333 2014-07-01 16:32:42,724.724 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:43,448.448 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 997 16 4645 4629 18.5682 4 0.737725 3.39275 2014-07-01 16:32:44,269.269 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:44,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 998 16 4646 4630 18.5536 4 2.85575 3.39264 2014-07-01 16:32:44,979.979 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:32:11.315194', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886953144, u'health': u'HEALTH_OK', u'kb_used': 9827908, u'store_stats': {u'bytes_total': 10401872, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'60 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'45 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 3009/10627 objects degraded (28.315%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 39954567, u'degraded_objects': 3009, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20332089700, u'degraded_total': 10627, u'bytes_used': 35247333376, u'recovering_objects_per_sec': 10, u'write_bytes_sec': 40502670, u'op_per_sec': 30, u'version': 857, u'pgs_by_state': [{u'count': 60, u'state_name': u'active+recovery_wait'}, {u'count': 161, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3605924, u'degraded_ratio': u'28.315', u'bytes_avail': 4963335204864}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 439, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:32:44,980.980 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:45,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 999 16 4646 4630 18.5351 0 - 3.39264 2014-07-01 16:32:45,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: finishing write tid 2 to mira0435601-98 2014-07-01 16:32:45,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: finishing write tid 3 to mira0435601-98 2014-07-01 16:32:45,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: finishing write tid 5 to mira0435601-98 2014-07-01 16:32:45,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: finishing write tid 1 to mira0435601-432 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: finishing write tid 3 to mira0435601-432 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: finishing write tid 1 to mira0435601-262 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: finishing write tid 3 to mira0435601-262 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: finishing write tid 6 to mira0435601-98 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 214 (ObjNum 984 snap 0 seq_num 984) dirty exists 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: finishing write tid 4 to mira0435601-262 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 216 (ObjNum 989 snap 0 seq_num 989) dirty exists 2014-07-01 16:32:45,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1401: expect (ObjNum 767 snap 0 seq_num 767) 2014-07-01 16:32:45,756.756 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:32:45,757.757 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:46,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:32:46.361867min lat: 0.057269 max lat: 68.3165 avg lat: 3.39262 2014-07-01 16:32:46,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:32:46,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1000 16 4648 4632 18.5245 4 2.93348 3.39262 2014-07-01 16:32:46,517.517 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:46,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: finishing write tid 4 to mira0435601-432 2014-07-01 16:32:46,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 94 (ObjNum 988 snap 0 seq_num 988) dirty exists 2014-07-01 16:32:46,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1394: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 16:32:47,448.448 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1001 16 4648 4632 18.506 0 - 3.39262 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 1 to mira0435601-322 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 2 to mira0435601-322 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 3 to mira0435601-322 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 4 to mira0435601-322 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 6 to mira0435601-322 2014-07-01 16:32:47,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: expect (ObjNum 858 snap 0 seq_num 858) 2014-07-01 16:32:48,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1002 16 4656 4640 18.5195 16 0.178515 3.39997 2014-07-01 16:32:49,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1003 16 4659 4643 18.513 12 0.29287 3.39798 2014-07-01 16:32:50,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1004 16 4660 4644 18.4985 4 1.00056 3.39747 2014-07-01 16:32:51,025.025 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:51,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 1 to mira0435601-117 2014-07-01 16:32:51,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 7 to mira0435601-322 2014-07-01 16:32:51,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 190 (ObjNum 987 snap 0 seq_num 987) dirty exists 2014-07-01 16:32:51,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 1 to mira0435601-333 2014-07-01 16:32:51,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1412: expect (ObjNum 707 snap 0 seq_num 707) 2014-07-01 16:32:51,449.449 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1005 16 4660 4644 18.4801 0 - 3.39747 2014-07-01 16:32:51,792.792 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:52,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 2 to mira0435601-117 2014-07-01 16:32:52,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 2 to mira0435601-333 2014-07-01 16:32:52,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 4 to mira0435601-333 2014-07-01 16:32:52,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: expect (ObjNum 82 snap 0 seq_num 82) 2014-07-01 16:32:52,450.450 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1006 16 4668 4652 18.4936 16 0.376926 3.40319 2014-07-01 16:32:53,332.332 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:53,450.450 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1007 16 4670 4654 18.4831 8 11.7312 3.40449 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 3 to mira0435601-117 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 4 to mira0435601-117 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 6 to mira0435601-117 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: finishing write tid 1 to mira0435601-45 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: finishing write tid 2 to mira0435601-45 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: finishing write tid 3 to mira0435601-45 2014-07-01 16:32:53,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: finishing write tid 5 to mira0435601-45 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1394: done (15 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: done (14 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: done (13 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: done (12 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1401: done (11 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: done (10 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: done (9 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1411: done (8 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1412: done (7 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: done (6 left) 2014-07-01 16:32:53,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: done (5 left) 2014-07-01 16:32:53,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 317 current snap is 0 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: seq_num 997 ranges {1052672=458752} 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: writing mira0435601-317 from 1052672 to 1511424 tid 1 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1418: read oid 111 snap -1 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: finishing write tid 7 to mira0435601-117 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 117 v 105 (ObjNum 996 snap 0 seq_num 996) dirty exists 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 5 to mira0435601-333 2014-07-01 16:32:53,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 188 (ObjNum 995 snap 0 seq_num 995) dirty exists 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: done (6 left) 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: done (5 left) 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1419: read oid 500 snap -1 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 154 current snap is 0 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: seq_num 998 ranges {2527232=487424,3014656=454656,3469312=643072,4112384=106496} 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: writing mira0435601-154 from 2527232 to 3014656 tid 1 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: writing mira0435601-154 from 3014656 to 3469312 tid 2 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: writing mira0435601-154 from 3469312 to 4112384 tid 3 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: writing mira0435601-154 from 4112384 to 4218880 tid 4 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1421: read oid 410 snap -1 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1419: done (7 left) 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1422: read oid 152 snap -1 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: read oid 352 snap -1 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: seq_num 999 ranges {7897088=761856,8658944=438272,9097216=364544} 2014-07-01 16:32:53,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: writing mira0435601-337 from 7897088 to 8658944 tid 1 2014-07-01 16:32:53,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: writing mira0435601-337 from 8658944 to 9097216 tid 2 2014-07-01 16:32:53,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: writing mira0435601-337 from 9097216 to 9461760 tid 3 2014-07-01 16:32:53,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:1425: read oid 215 snap -1 2014-07-01 16:32:53,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:1421: expect (ObjNum 738 snap 0 seq_num 738) 2014-07-01 16:32:54,032.032 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:31:54.532097', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886967000, u'health': u'HEALTH_OK', u'kb_used': 9814052, u'store_stats': {u'bytes_total': 9353281, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:32:11.315194', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886953144, u'health': u'HEALTH_OK', u'kb_used': 9827908, u'store_stats': {u'bytes_total': 10401872, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'71 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 3015/10660 objects degraded (28.283%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3272373, u'degraded_objects': 3015, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20392956222, u'degraded_total': 10660, u'bytes_used': 35582558208, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 8193717, u'op_per_sec': 5, u'version': 864, u'pgs_by_state': [{u'count': 59, u'state_name': u'active+recovery_wait'}, {u'count': 162, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2597020, u'degraded_ratio': u'28.283', u'bytes_avail': 4962999980032}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 441, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:32:54,033.033 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:32:54,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1421: done (10 left) 2014-07-01 16:32:54,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1426: read oid 362 snap -1 2014-07-01 16:32:54,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: read oid 159 snap -1 2014-07-01 16:32:54,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 16:32:54,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: seq_num 1000 ranges {122880=630784,753664=466944,1220608=335872} 2014-07-01 16:32:54,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: writing mira0435601-48 from 122880 to 753664 tid 1 2014-07-01 16:32:54,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: writing mira0435601-48 from 753664 to 1220608 tid 2 2014-07-01 16:32:54,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: writing mira0435601-48 from 1220608 to 1556480 tid 3 2014-07-01 16:32:54,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: delete oid 126 current snap is 0 2014-07-01 16:32:54,451.451 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1008 16 4670 4654 18.4648 0 - 3.40449 2014-07-01 16:32:54,872.872 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:32:54,872.872 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:32:55,450.450 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1009 16 4670 4654 18.4465 0 - 3.40449 2014-07-01 16:32:55,648.648 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:32:56,450.450 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1010 16 4670 4654 18.4282 0 - 3.40449 2014-07-01 16:32:57,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1011 16 4670 4654 18.41 0 - 3.40449 2014-07-01 16:32:58,451.451 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1012 16 4670 4654 18.3918 0 - 3.40449 2014-07-01 16:32:58,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: finishing write tid 1 to mira0435601-337 2014-07-01 16:32:58,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: finishing write tid 2 to mira0435601-337 2014-07-01 16:32:58,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: finishing write tid 3 to mira0435601-337 2014-07-01 16:32:58,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: finishing write tid 5 to mira0435601-337 2014-07-01 16:32:58,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: finishing write tid 1 to mira0435601-48 2014-07-01 16:32:59,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1013 16 4672 4656 18.3815 1.33333 5.4104 3.40531 2014-07-01 16:32:59,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: expect (ObjNum 351 snap 0 seq_num 351) 2014-07-01 16:33:00,017.017 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:00,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1014 16 4675 4659 18.3753 12 0.72917 3.40646 2014-07-01 16:33:00,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1426: expect (ObjNum 948 snap 0 seq_num 948) 2014-07-01 16:33:00,765.765 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:01,451.451 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1015 16 4690 4674 18.4162 60 0.49992 3.42525 2014-07-01 16:33:02,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: finishing write tid 2 to mira0435601-48 2014-07-01 16:33:02,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: finishing write tid 3 to mira0435601-48 2014-07-01 16:33:02,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: finishing write tid 5 to mira0435601-48 2014-07-01 16:33:02,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: finishing write tid 6 to mira0435601-337 2014-07-01 16:33:02,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 270 (ObjNum 999 snap 0 seq_num 999) dirty exists 2014-07-01 16:33:02,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: finishing write tid 6 to mira0435601-48 2014-07-01 16:33:02,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 263 (ObjNum 1000 snap 0 seq_num 1000) dirty exists 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: finishing write tid 6 to mira0435601-45 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 159 (ObjNum 975 snap 0 seq_num 975) dirty exists 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: finishing write tid 1 to mira0435601-217 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: finishing write tid 2 to mira0435601-217 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: finishing write tid 3 to mira0435601-217 2014-07-01 16:33:02,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: finishing write tid 5 to mira0435601-217 2014-07-01 16:33:02,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1425: expect (ObjNum 844 snap 0 seq_num 844) 2014-07-01 16:33:02,301.301 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:02,451.451 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1016 16 4697 4681 18.4257 28 0.110199 3.42064 2014-07-01 16:33:03,001.001 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:32:11.315194', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886953144, u'health': u'HEALTH_OK', u'kb_used': 9827908, u'store_stats': {u'bytes_total': 10401872, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061824}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'25 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'56 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'71 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'15 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 3002/10704 objects degraded (28.046%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 32533561, u'degraded_objects': 3002, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20488012075, u'degraded_total': 10704, u'bytes_used': 35951603712, u'recovering_objects_per_sec': 7, u'write_bytes_sec': 33728947, u'op_per_sec': 14, u'version': 871, u'pgs_by_state': [{u'count': 56, u'state_name': u'active+recovery_wait'}, {u'count': 163, u'state_name': u'active+clean'}, {u'count': 25, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2472185, u'degraded_ratio': u'28.046', u'bytes_avail': 4962630934528}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 444, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:03,001.001 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:03,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1017 16 4707 4691 18.4469 40 0.925219 3.41969 2014-07-01 16:33:03,795.795 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:33:03,795.795 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:04,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1018 16 4712 4696 18.4484 20 0.20842 3.41709 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 1 to mira0435601-317 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 3 to mira0435601-317 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 1 to mira0435601-154 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 2 to mira0435601-154 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 3 to mira0435601-154 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 4 to mira0435601-154 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 6 to mira0435601-154 2014-07-01 16:33:04,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 4 to mira0435601-317 2014-07-01 16:33:04,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 162 (ObjNum 997 snap 0 seq_num 997) dirty exists 2014-07-01 16:33:04,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: finishing write tid 7 to mira0435601-154 2014-07-01 16:33:04,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 217 (ObjNum 998 snap 0 seq_num 998) dirty exists 2014-07-01 16:33:04,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: finishing write tid 6 to mira0435601-217 2014-07-01 16:33:04,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 178 (ObjNum 985 snap 0 seq_num 985) dirty exists 2014-07-01 16:33:04,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: expect (ObjNum 158 snap 0 seq_num 158) 2014-07-01 16:33:05,333.333 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:05,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1019 16 4713 4697 18.4342 4 1.73489 3.41673 2014-07-01 16:33:06,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:33:06.365307min lat: 0.057269 max lat: 68.3165 avg lat: 3.41673 2014-07-01 16:33:06,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:33:06,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1020 16 4713 4697 18.4162 0 - 3.41673 2014-07-01 16:33:07,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1021 16 4713 4697 18.3981 0 - 3.41673 2014-07-01 16:33:08,452.452 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1022 16 4713 4697 18.3801 0 - 3.41673 2014-07-01 16:33:09,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1023 16 4713 4697 18.3621 0 - 3.41673 2014-07-01 16:33:10,186.186 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:10,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1024 16 4713 4697 18.3442 0 - 3.41673 2014-07-01 16:33:10,954.954 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:11,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1025 16 4719 4703 18.3497 4 0.183427 3.41814 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: done (13 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: done (12 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: done (11 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1418: done (10 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: done (9 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1422: done (8 left) 2014-07-01 16:33:11,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: done (7 left) 2014-07-01 16:33:11,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: done (6 left) 2014-07-01 16:33:11,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:1425: done (5 left) 2014-07-01 16:33:11,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:1426: done (4 left) 2014-07-01 16:33:11,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: done (3 left) 2014-07-01 16:33:11,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: done (2 left) 2014-07-01 16:33:11,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: done (1 left) 2014-07-01 16:33:11,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: delete oid 323 current snap is 0 2014-07-01 16:33:11,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 1 to mira0435601-3 2014-07-01 16:33:11,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 2 to mira0435601-3 2014-07-01 16:33:11,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 3 to mira0435601-3 2014-07-01 16:33:11,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 4 to mira0435601-3 2014-07-01 16:33:11,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 6 to mira0435601-3 2014-07-01 16:33:11,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: done (1 left) 2014-07-01 16:33:11,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 317 current snap is 0 2014-07-01 16:33:11,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: seq_num 1003 ranges {1511424=450560,1961984=401408} 2014-07-01 16:33:12,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: writing mira0435601-317 from 1511424 to 1961984 tid 1 2014-07-01 16:33:12,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: writing mira0435601-317 from 1961984 to 2363392 tid 2 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1432: read oid 57 snap -1 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1433: read oid 134 snap -1 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1434: read oid 378 snap -1 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: seq_num 1004 ranges {0=479232,479232=446464,925696=589824,1515520=569344} 2014-07-01 16:33:12,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: writing mira0435601-9 from 0 to 479232 tid 1 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: writing mira0435601-9 from 479232 to 925696 tid 2 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: writing mira0435601-9 from 925696 to 1515520 tid 3 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: writing mira0435601-9 from 1515520 to 2084864 tid 4 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1436: read oid 77 snap -1 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 82 current snap is 0 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: seq_num 1005 ranges {1179648=753664,1933312=430080,2363392=659456,3022848=114688} 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: writing mira0435601-82 from 1179648 to 1933312 tid 1 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: writing mira0435601-82 from 1933312 to 2363392 tid 2 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: writing mira0435601-82 from 2363392 to 3022848 tid 3 2014-07-01 16:33:12,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: writing mira0435601-82 from 3022848 to 3137536 tid 4 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 283 current snap is 0 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: seq_num 1006 ranges {933888=270336} 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: writing mira0435601-283 from 933888 to 1204224 tid 1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: read oid 177 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: read oid 374 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: read oid 25 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1442: read oid 351 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1443: read oid 195 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1444: read oid 411 snap -1 2014-07-01 16:33:12,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1445: delete oid 290 current snap is 0 2014-07-01 16:33:12,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:1433: expect (ObjNum 133 snap 0 seq_num 133) 2014-07-01 16:33:12,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1026 16 4736 4720 18.3981 68 0.561339 3.42805 2014-07-01 16:33:12,501.501 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:13,204.204 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'25 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'49 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'73 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2944/10789 objects degraded (27.287%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 41527818, u'degraded_objects': 2944, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20677648626, u'degraded_total': 10789, u'bytes_used': 36637732864, u'recovering_objects_per_sec': 9, u'write_bytes_sec': 28246216, u'op_per_sec': 15, u'version': 877, u'pgs_by_state': [{u'count': 49, u'state_name': u'active+recovery_wait'}, {u'count': 170, u'state_name': u'active+clean'}, {u'count': 25, u'state_name': u'active+recovering'}], u'read_bytes_sec': 4435649, u'degraded_ratio': u'27.287', u'bytes_avail': 4961944805376}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 446, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:13,205.205 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:13,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1027 16 4741 4725 18.3997 20 1.83116 3.4263 2014-07-01 16:33:13,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: expect (ObjNum 779 snap 0 seq_num 779) 2014-07-01 16:33:13,962.962 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:33:13,962.962 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:14,453.453 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1028 16 4747 4731 18.4051 24 2.11929 3.42373 2014-07-01 16:33:14,704.704 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:15,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1029 16 4747 4731 18.3872 0 - 3.42373 2014-07-01 16:33:16,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:1442: expect (ObjNum 884 snap 0 seq_num 884) 2014-07-01 16:33:16,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1030 16 4747 4731 18.3694 0 - 3.42373 2014-07-01 16:33:17,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1031 16 4750 4734 18.3632 4 3.41422 3.42373 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 1 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 2 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 3 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 4 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 6 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 1 to mira0435601-82 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 2 to mira0435601-82 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 3 to mira0435601-82 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 4 to mira0435601-82 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 6 to mira0435601-82 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: finishing write tid 7 to mira0435601-9 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 268 (ObjNum 1004 snap 0 seq_num 1004) dirty exists 2014-07-01 16:33:17,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: finishing write tid 7 to mira0435601-82 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 273 (ObjNum 1005 snap 0 seq_num 1005) dirty exists 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: finishing write tid 1 to mira0435601-283 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: finishing write tid 3 to mira0435601-283 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: finishing write tid 4 to mira0435601-283 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 272 (ObjNum 1006 snap 0 seq_num 1006) dirty exists 2014-07-01 16:33:17,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1443: expect (ObjNum 944 snap 0 seq_num 944) 2014-07-01 16:33:18,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: expect (ObjNum 949 snap 0 seq_num 949) 2014-07-01 16:33:18,454.454 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1032 16 4751 4735 18.3493 4 0.612778 3.42313 2014-07-01 16:33:19,078.078 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:19,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1033 16 4753 4737 18.3393 8 5.75131 3.42473 2014-07-01 16:33:19,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: finishing write tid 7 to mira0435601-3 2014-07-01 16:33:19,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 3 v 164 (ObjNum 992 snap 0 seq_num 992) dirty exists 2014-07-01 16:33:19,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: finishing write tid 1 to mira0435601-317 2014-07-01 16:33:19,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1434: expect (ObjNum 377 snap 0 seq_num 377) 2014-07-01 16:33:19,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: finishing write tid 2 to mira0435601-317 2014-07-01 16:33:19,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: finishing write tid 4 to mira0435601-317 2014-07-01 16:33:19,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: finishing write tid 5 to mira0435601-317 2014-07-01 16:33:19,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 165 (ObjNum 1003 snap 0 seq_num 1003) dirty exists 2014-07-01 16:33:19,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1432: expect (ObjNum 860 snap 0 seq_num 860) 2014-07-01 16:33:20,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1034 16 4760 4744 18.3486 28 0.477042 3.4274 2014-07-01 16:33:20,656.656 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:21,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1035 16 4760 4744 18.3309 0 - 3.4274 2014-07-01 16:33:22,366.366 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: done (15 left) 2014-07-01 16:33:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: done (14 left) 2014-07-01 16:33:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1432: done (13 left) 2014-07-01 16:33:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1433: done (12 left) 2014-07-01 16:33:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1434: done (11 left) 2014-07-01 16:33:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: done (10 left) 2014-07-01 16:33:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: done (9 left) 2014-07-01 16:33:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: done (8 left) 2014-07-01 16:33:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: done (7 left) 2014-07-01 16:33:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: done (6 left) 2014-07-01 16:33:22,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: done (5 left) 2014-07-01 16:33:22,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:1442: done (4 left) 2014-07-01 16:33:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1443: done (3 left) 2014-07-01 16:33:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1445: done (2 left) 2014-07-01 16:33:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 98 current snap is 0 2014-07-01 16:33:22,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: seq_num 1008 ranges {2621440=327680} 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: writing mira0435601-98 from 2621440 to 2949120 tid 1 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 484 current snap is 0 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: seq_num 1009 ranges {172032=442368,614400=413696,1028096=389120} 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: writing mira0435601-484 from 172032 to 614400 tid 1 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: writing mira0435601-484 from 614400 to 1028096 tid 2 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: writing mira0435601-484 from 1028096 to 1417216 tid 3 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1448: read oid 73 snap -1 2014-07-01 16:33:22,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1436: expect (ObjNum 76 snap 0 seq_num 76) 2014-07-01 16:33:22,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1036 16 4765 4749 18.3325 10 10.7651 3.43174 2014-07-01 16:33:23,096.096 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'47 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'72 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2926/10835 objects degraded (27.005%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 22254801, u'degraded_objects': 2926, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 20776734943, u'degraded_total': 10835, u'bytes_used': 36980023296, u'recovering_objects_per_sec': 5, u'write_bytes_sec': 25904621, u'op_per_sec': 10, u'version': 881, u'pgs_by_state': [{u'count': 47, u'state_name': u'active+recovery_wait'}, {u'count': 171, u'state_name': u'active+clean'}, {u'count': 26, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1140314, u'degraded_ratio': u'27.005', u'bytes_avail': 4961602514944}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 446, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:23,097.097 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:23,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1037 16 4781 4765 18.3765 64 3.80757 3.42867 2014-07-01 16:33:23,834.834 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:33:23,835.835 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:24,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1444: expect (ObjNum 601 snap 0 seq_num 601) 2014-07-01 16:33:24,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1038 16 4781 4765 18.3588 0 - 3.42867 2014-07-01 16:33:24,638.638 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:25,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1039 16 4785 4769 18.3565 8 1.71836 3.42715 2014-07-01 16:33:25,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: finishing write tid 1 to mira0435601-98 2014-07-01 16:33:25,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: finishing write tid 3 to mira0435601-98 2014-07-01 16:33:25,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:1448: expect (ObjNum 72 snap 0 seq_num 72) 2014-07-01 16:33:26,455.455 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:33:26.368857min lat: 0.057269 max lat: 68.3165 avg lat: 3.42663 2014-07-01 16:33:26,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:33:26,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1040 16 4786 4770 18.3427 4 0.974197 3.42663 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: finishing write tid 1 to mira0435601-484 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: finishing write tid 2 to mira0435601-484 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: finishing write tid 3 to mira0435601-484 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: finishing write tid 5 to mira0435601-484 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1436: done (4 left) 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1444: done (3 left) 2014-07-01 16:33:26,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1448: done (2 left) 2014-07-01 16:33:26,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 16:33:26,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: seq_num 1010 ranges {1257472=520192,1777664=401408} 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: writing mira0435601-418 from 1257472 to 1777664 tid 1 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: writing mira0435601-418 from 1777664 to 2179072 tid 2 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: read oid 153 snap -1 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: seq_num 1011 ranges {2457600=544768,3002368=471040,3473408=655360,4128768=479232} 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: writing mira0435601-97 from 2457600 to 3002368 tid 1 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: writing mira0435601-97 from 3002368 to 3473408 tid 2 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: writing mira0435601-97 from 3473408 to 4128768 tid 3 2014-07-01 16:33:26,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: writing mira0435601-97 from 4128768 to 4608000 tid 4 2014-07-01 16:33:26,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 16:33:26,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: seq_num 1012 ranges {2596864=663552} 2014-07-01 16:33:26,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: writing mira0435601-288 from 2596864 to 3260416 tid 1 2014-07-01 16:33:26,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1453: delete oid 178 current snap is 0 2014-07-01 16:33:27,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: finishing write tid 1 to mira0435601-288 2014-07-01 16:33:27,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: finishing write tid 3 to mira0435601-288 2014-07-01 16:33:27,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1041 16 4787 4771 18.3289 4 0.764734 3.42607 2014-07-01 16:33:27,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: finishing write tid 4 to mira0435601-288 2014-07-01 16:33:27,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 275 (ObjNum 1012 snap 0 seq_num 1012) dirty exists 2014-07-01 16:33:28,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1042 16 4787 4771 18.3113 0 - 3.42607 2014-07-01 16:33:29,009.009 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:29,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1043 16 4787 4771 18.2938 0 - 3.42607 2014-07-01 16:33:29,767.767 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:30,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1044 16 4787 4771 18.2763 0 - 3.42607 2014-07-01 16:33:31,313.313 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:31,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1045 16 4787 4771 18.2588 0 - 3.42607 2014-07-01 16:33:31,996.996 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:32:39.394329', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883616100, u'health': u'HEALTH_OK', u'kb_used': 13164952, u'store_stats': {u'bytes_total': 10402193, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3062145}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'71 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2905/10895 objects degraded (26.664%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2905, u'num_pgs': 244, u'data_bytes': 20904546470, u'degraded_total': 10895, u'bytes_used': 37107007488, u'version': 892, u'pgs_by_state': [{u'count': 46, u'state_name': u'active+recovery_wait'}, {u'count': 172, u'state_name': u'active+clean'}, {u'count': 26, u'state_name': u'active+recovering'}], u'degraded_ratio': u'26.664', u'bytes_avail': 4961475530752}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 452, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:31,997.997 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:32,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: finishing write tid 4 to mira0435601-98 2014-07-01 16:33:32,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 218 (ObjNum 1008 snap 0 seq_num 1008) dirty exists 2014-07-01 16:33:32,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1046 16 4790 4774 18.2528 2.4 12.6252 3.43194 2014-07-01 16:33:32,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: finishing write tid 6 to mira0435601-484 2014-07-01 16:33:32,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 221 (ObjNum 1009 snap 0 seq_num 1009) dirty exists 2014-07-01 16:33:32,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: expect (ObjNum 152 snap 0 seq_num 152) 2014-07-01 16:33:32,779.779 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:33:32,779.779 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:33,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1047 16 4801 4785 18.2774 44 9.53936 3.44252 2014-07-01 16:33:33,572.572 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:34,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: finishing write tid 1 to mira0435601-418 2014-07-01 16:33:34,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: finishing write tid 2 to mira0435601-418 2014-07-01 16:33:34,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: finishing write tid 4 to mira0435601-418 2014-07-01 16:33:34,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 1 to mira0435601-97 2014-07-01 16:33:34,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 2 to mira0435601-97 2014-07-01 16:33:34,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 3 to mira0435601-97 2014-07-01 16:33:34,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 4 to mira0435601-97 2014-07-01 16:33:34,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 6 to mira0435601-97 2014-07-01 16:33:34,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: done (6 left) 2014-07-01 16:33:34,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: done (5 left) 2014-07-01 16:33:34,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: done (4 left) 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: done (3 left) 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1453: done (2 left) 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 119 current snap is 0 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: seq_num 1014 ranges {98304=561152,659456=466944,1126400=425984,1552384=311296} 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: writing mira0435601-119 from 98304 to 659456 tid 1 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: writing mira0435601-119 from 659456 to 1126400 tid 2 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: writing mira0435601-119 from 1126400 to 1552384 tid 3 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: writing mira0435601-119 from 1552384 to 1863680 tid 4 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 25 current snap is 0 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: seq_num 1015 ranges {0=81920} 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: writing mira0435601-25 from 0 to 81920 tid 1 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 144 current snap is 0 2014-07-01 16:33:34,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: seq_num 1016 ranges {299008=512000,811008=684032,1495040=118784} 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0435601-144 from 299008 to 811008 tid 1 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0435601-144 from 811008 to 1495040 tid 2 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0435601-144 from 1495040 to 1613824 tid 3 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1457: read oid 8 snap -1 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1458: read oid 87 snap -1 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: read oid 398 snap -1 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 50 current snap is 0 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: seq_num 1017 ranges {1294336=782336,2076672=401408} 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: writing mira0435601-50 from 1294336 to 2076672 tid 1 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: writing mira0435601-50 from 2076672 to 2478080 tid 2 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 486 current snap is 0 2014-07-01 16:33:34,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: seq_num 1018 ranges {1273856=655360,1929216=401408} 2014-07-01 16:33:34,591.591 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1048 16 4807 4791 18.2828 24 0.135084 3.44132 2014-07-01 16:33:34,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: writing mira0435601-486 from 1273856 to 1929216 tid 1 2014-07-01 16:33:34,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: writing mira0435601-486 from 1929216 to 2330624 tid 2 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 222 current snap is 0 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: seq_num 1019 ranges {1978368=733184} 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: writing mira0435601-222 from 1978368 to 2711552 tid 1 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 443 current snap is 0 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: seq_num 1020 ranges {1892352=602112} 2014-07-01 16:33:34,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: writing mira0435601-443 from 1892352 to 2494464 tid 1 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1464: read oid 114 snap -1 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: seq_num 1021 ranges {372736=630784,1003520=442368,1445888=651264,2097152=110592} 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: writing mira0435601-31 from 372736 to 1003520 tid 1 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: writing mira0435601-31 from 1003520 to 1445888 tid 2 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: writing mira0435601-31 from 1445888 to 2097152 tid 3 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: writing mira0435601-31 from 2097152 to 2207744 tid 4 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: read oid 278 snap -1 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 191 current snap is 0 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: seq_num 1022 ranges {884736=503808,1388544=675840,2064384=475136,2539520=53248} 2014-07-01 16:33:34,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0435601-191 from 884736 to 1388544 tid 1 2014-07-01 16:33:34,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0435601-191 from 1388544 to 2064384 tid 2 2014-07-01 16:33:34,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0435601-191 from 2064384 to 2539520 tid 3 2014-07-01 16:33:34,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0435601-191 from 2539520 to 2592768 tid 4 2014-07-01 16:33:34,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:33:35,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1049 16 4807 4791 18.2654 0 - 3.44132 2014-07-01 16:33:36,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 1 to mira0435601-443 2014-07-01 16:33:36,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 3 to mira0435601-443 2014-07-01 16:33:36,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 4 to mira0435601-443 2014-07-01 16:33:36,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 219 (ObjNum 1020 snap 0 seq_num 1020) dirty exists 2014-07-01 16:33:36,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: done (15 left) 2014-07-01 16:33:36,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 371 current snap is 0 2014-07-01 16:33:36,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: seq_num 1023 ranges {0=774144,774144=659456,1433600=143360} 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: writing mira0435601-371 from 0 to 774144 tid 1 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: writing mira0435601-371 from 774144 to 1433600 tid 2 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: writing mira0435601-371 from 1433600 to 1576960 tid 3 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: finishing write tid 5 to mira0435601-418 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 174 (ObjNum 1010 snap 0 seq_num 1010) dirty exists 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: done (15 left) 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1469: delete oid 315 current snap is 0 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: finishing write tid 7 to mira0435601-97 2014-07-01 16:33:36,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 179 (ObjNum 1011 snap 0 seq_num 1011) dirty exists 2014-07-01 16:33:36,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: expect (ObjNum 901 snap 0 seq_num 901) 2014-07-01 16:33:36,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1050 16 4819 4803 18.2937 24 2.01678 3.44244 2014-07-01 16:33:37,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: expect (ObjNum 397 snap 0 seq_num 397) 2014-07-01 16:33:37,457.457 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1051 16 4821 4805 18.2839 8 3.8339 3.44186 2014-07-01 16:33:37,972.972 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:38,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1052 16 4821 4805 18.2665 0 - 3.44186 2014-07-01 16:33:38,531.531 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:39,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:1464: expect (ObjNum 113 snap 0 seq_num 113) 2014-07-01 16:33:39,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1053 16 4822 4806 18.253 2 2.8084 3.44173 2014-07-01 16:33:39,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:1457: expect (ObjNum 7 snap 0 seq_num 7) 2014-07-01 16:33:39,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1458: expect (ObjNum 86 snap 0 seq_num 86) 2014-07-01 16:33:40,267.267 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:40,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1054 16 4822 4806 18.2357 0 - 3.44173 2014-07-01 16:33:41,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1055 16 4822 4806 18.2184 0 - 3.44173 2014-07-01 16:33:41,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: finishing write tid 1 to mira0435601-50 2014-07-01 16:33:41,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: finishing write tid 1 to mira0435601-371 2014-07-01 16:33:41,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: finishing write tid 2 to mira0435601-371 2014-07-01 16:33:41,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: finishing write tid 3 to mira0435601-371 2014-07-01 16:33:41,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: finishing write tid 5 to mira0435601-371 2014-07-01 16:33:41,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: finishing write tid 2 to mira0435601-50 2014-07-01 16:33:41,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 1 to mira0435601-119 2014-07-01 16:33:41,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: finishing write tid 4 to mira0435601-50 2014-07-01 16:33:41,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 2 to mira0435601-119 2014-07-01 16:33:41,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 3 to mira0435601-119 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 4 to mira0435601-119 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: finishing write tid 1 to mira0435601-25 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 6 to mira0435601-119 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: finishing write tid 3 to mira0435601-25 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 1 to mira0435601-191 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: finishing write tid 6 to mira0435601-371 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 279 (ObjNum 1023 snap 0 seq_num 1023) dirty exists 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 2 to mira0435601-191 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 3 to mira0435601-191 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 4 to mira0435601-191 2014-07-01 16:33:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 6 to mira0435601-191 2014-07-01 16:33:41,896.896 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'69 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2888/10964 objects degraded (26.341%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3694728, u'degraded_objects': 2888, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21049937991, u'degraded_total': 10964, u'bytes_used': 37339283456, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 17458555, u'op_per_sec': 9, u'version': 900, u'pgs_by_state': [{u'count': 46, u'state_name': u'active+recovery_wait'}, {u'count': 174, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 633828, u'degraded_ratio': u'26.341', u'bytes_avail': 4961243254784}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 455, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:41,896.896 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:42,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1056 16 4830 4814 18.2314 10.6667 0.12297 3.4406 2014-07-01 16:33:42,480.480 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:33:42,480.480 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:43,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 1 to mira0435601-144 2014-07-01 16:33:43,430.430 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:43,458.458 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1057 16 4837 4821 18.2407 28 7.11591 3.44847 2014-07-01 16:33:43,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 2 to mira0435601-144 2014-07-01 16:33:43,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 3 to mira0435601-144 2014-07-01 16:33:44,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 5 to mira0435601-144 2014-07-01 16:33:44,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 1 to mira0435601-222 2014-07-01 16:33:44,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 3 to mira0435601-222 2014-07-01 16:33:44,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: finishing write tid 5 to mira0435601-50 2014-07-01 16:33:44,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 156 (ObjNum 1017 snap 0 seq_num 1017) dirty exists 2014-07-01 16:33:44,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 6 to mira0435601-144 2014-07-01 16:33:44,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 84 (ObjNum 1016 snap 0 seq_num 1016) dirty exists 2014-07-01 16:33:44,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 1 to mira0435601-31 2014-07-01 16:33:44,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 2 to mira0435601-31 2014-07-01 16:33:44,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 4 to mira0435601-222 2014-07-01 16:33:44,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 197 (ObjNum 1019 snap 0 seq_num 1019) dirty exists 2014-07-01 16:33:44,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 7 to mira0435601-191 2014-07-01 16:33:44,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 170 (ObjNum 1022 snap 0 seq_num 1022) dirty exists 2014-07-01 16:33:44,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: finishing write tid 7 to mira0435601-119 2014-07-01 16:33:44,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 184 (ObjNum 1014 snap 0 seq_num 1014) dirty exists 2014-07-01 16:33:44,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: finishing write tid 4 to mira0435601-25 2014-07-01 16:33:44,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 220 (ObjNum 1015 snap 0 seq_num 1015) dirty exists 2014-07-01 16:33:44,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1058 16 4856 4840 18.2952 76 0.42205 3.44397 2014-07-01 16:33:45,459.459 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1059 16 4858 4842 18.2855 8 1.04211 3.44286 2014-07-01 16:33:46,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:33:46.372748min lat: 0.057269 max lat: 68.3165 avg lat: 3.44286 2014-07-01 16:33:46,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:33:46,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1060 16 4858 4842 18.2683 0 - 3.44286 2014-07-01 16:33:47,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1061 16 4858 4842 18.2511 0 - 3.44286 2014-07-01 16:33:47,959.959 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:48,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1062 16 4858 4842 18.2339 0 - 3.44286 2014-07-01 16:33:48,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 3 to mira0435601-31 2014-07-01 16:33:48,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 4 to mira0435601-31 2014-07-01 16:33:48,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 6 to mira0435601-31 2014-07-01 16:33:48,669.669 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:48,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: finishing write tid 1 to mira0435601-486 2014-07-01 16:33:49,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1063 16 4867 4851 18.2506 9 0.618987 3.44326 2014-07-01 16:33:50,210.210 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:50,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1064 16 4872 4856 18.2522 20 0.717812 3.44047 2014-07-01 16:33:50,915.915 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:32:54.532375', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886922084, u'health': u'HEALTH_OK', u'kb_used': 9858968, u'store_stats': {u'bytes_total': 14596161, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3061809}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'45 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'67 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2860/11049 objects degraded (25.885%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12226896, u'degraded_objects': 2860, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21233442773, u'degraded_total': 11049, u'bytes_used': 37757276160, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 19213694, u'op_per_sec': 4, u'version': 912, u'pgs_by_state': [{u'count': 45, u'state_name': u'active+recovery_wait'}, {u'count': 176, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'25.885', u'bytes_avail': 4960825262080}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 462, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:50,916.916 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:51,461.461 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1065 16 4872 4856 18.2351 0 - 3.44047 2014-07-01 16:33:51,682.682 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:33:51,682.682 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:52,461.461 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1066 16 4872 4856 18.218 0 - 3.44047 2014-07-01 16:33:52,482.482 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:53,460.460 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1067 16 4874 4858 18.2084 2.66667 4.07388 3.44066 2014-07-01 16:33:54,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: finishing write tid 2 to mira0435601-486 2014-07-01 16:33:54,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: finishing write tid 4 to mira0435601-486 2014-07-01 16:33:54,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: done (15 left) 2014-07-01 16:33:54,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: done (14 left) 2014-07-01 16:33:54,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: done (13 left) 2014-07-01 16:33:54,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: done (12 left) 2014-07-01 16:33:54,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1457: done (11 left) 2014-07-01 16:33:54,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1458: done (10 left) 2014-07-01 16:33:54,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: done (9 left) 2014-07-01 16:33:54,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: done (8 left) 2014-07-01 16:33:54,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: done (7 left) 2014-07-01 16:33:54,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1464: done (6 left) 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: done (5 left) 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: done (4 left) 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: done (3 left) 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1469: done (2 left) 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 25 current snap is 0 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: seq_num 1025 ranges {81920=724992,806912=401408} 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: writing mira0435601-25 from 81920 to 806912 tid 1 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: writing mira0435601-25 from 806912 to 1208320 tid 2 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: read oid 16 snap -1 2014-07-01 16:33:54,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: read oid 68 snap -1 2014-07-01 16:33:54,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: delete oid 452 current snap is 0 2014-07-01 16:33:54,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: done (5 left) 2014-07-01 16:33:54,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 141 current snap is 0 2014-07-01 16:33:54,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: seq_num 1027 ranges {372736=622592,995328=638976,1634304=593920,2228224=372736} 2014-07-01 16:33:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0435601-141 from 372736 to 995328 tid 1 2014-07-01 16:33:54,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0435601-141 from 995328 to 1634304 tid 2 2014-07-01 16:33:54,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1068 16 4881 4865 18.2176 28 10.1547 3.44639 2014-07-01 16:33:54,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0435601-141 from 1634304 to 2228224 tid 3 2014-07-01 16:33:54,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0435601-141 from 2228224 to 2600960 tid 4 2014-07-01 16:33:54,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: read oid 413 snap -1 2014-07-01 16:33:54,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: read oid 243 snap -1 2014-07-01 16:33:54,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: read oid 310 snap -1 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1478: delete oid 305 current snap is 0 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1478: done (9 left) 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1479: read oid 130 snap -1 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: seq_num 1029 ranges {2166784=413696} 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: writing mira0435601-439 from 2166784 to 2580480 tid 1 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 111 current snap is 0 2014-07-01 16:33:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: seq_num 1030 ranges {0=442368,442368=634880,1077248=569344} 2014-07-01 16:33:54,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: writing mira0435601-111 from 0 to 442368 tid 1 2014-07-01 16:33:54,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: writing mira0435601-111 from 442368 to 1077248 tid 2 2014-07-01 16:33:54,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: writing mira0435601-111 from 1077248 to 1646592 tid 3 2014-07-01 16:33:54,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 357 current snap is 0 2014-07-01 16:33:54,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: seq_num 1031 ranges {421888=712704,1134592=622592,1757184=180224} 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: writing mira0435601-357 from 421888 to 1134592 tid 1 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: writing mira0435601-357 from 1134592 to 1757184 tid 2 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: writing mira0435601-357 from 1757184 to 1937408 tid 3 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 230 current snap is 0 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: seq_num 1032 ranges {1527808=585728,2113536=696320,2809856=106496} 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: writing mira0435601-230 from 1527808 to 2113536 tid 1 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: writing mira0435601-230 from 2113536 to 2809856 tid 2 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: writing mira0435601-230 from 2809856 to 2916352 tid 3 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 496 current snap is 0 2014-07-01 16:33:54,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: seq_num 1033 ranges {2318336=663552,2981888=401408} 2014-07-01 16:33:54,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: writing mira0435601-496 from 2318336 to 2981888 tid 1 2014-07-01 16:33:54,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: writing mira0435601-496 from 2981888 to 3383296 tid 2 2014-07-01 16:33:54,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 366 current snap is 0 2014-07-01 16:33:54,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: seq_num 1034 ranges {0=688128,688128=462848,1150976=663552,1814528=479232} 2014-07-01 16:33:54,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: writing mira0435601-366 from 0 to 688128 tid 1 2014-07-01 16:33:54,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: writing mira0435601-366 from 688128 to 1150976 tid 2 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: writing mira0435601-366 from 1150976 to 1814528 tid 3 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: writing mira0435601-366 from 1814528 to 2293760 tid 4 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: finishing write tid 1 to mira0435601-357 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: finishing write tid 2 to mira0435601-357 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: finishing write tid 3 to mira0435601-357 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: finishing write tid 5 to mira0435601-357 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: finishing write tid 5 to mira0435601-486 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 167 (ObjNum 1018 snap 0 seq_num 1018) dirty exists 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: finishing write tid 6 to mira0435601-357 2014-07-01 16:33:54,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 276 (ObjNum 1031 snap 0 seq_num 1031) dirty exists 2014-07-01 16:33:54,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1479: expect (ObjNum 129 snap 0 seq_num 129) 2014-07-01 16:33:55,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: expect (ObjNum 696 snap 0 seq_num 696) 2014-07-01 16:33:55,461.461 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1069 16 4896 4880 18.2566 60 0.588309 3.44729 2014-07-01 16:33:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: finishing write tid 7 to mira0435601-31 2014-07-01 16:33:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 196 (ObjNum 1021 snap 0 seq_num 1021) dirty exists 2014-07-01 16:33:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: done (15 left) 2014-07-01 16:33:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: done (14 left) 2014-07-01 16:33:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: done (13 left) 2014-07-01 16:33:56,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1479: done (12 left) 2014-07-01 16:33:56,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: done (11 left) 2014-07-01 16:33:56,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: read oid 302 snap -1 2014-07-01 16:33:56,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 279 current snap is 0 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: seq_num 1035 ranges {1904640=405504,2310144=401408} 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: writing mira0435601-279 from 1904640 to 2310144 tid 1 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: writing mira0435601-279 from 2310144 to 2711552 tid 2 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 300 current snap is 0 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: seq_num 1036 ranges {0=434176,434176=675840,1110016=610304,1720320=319488} 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: writing mira0435601-300 from 0 to 434176 tid 1 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: writing mira0435601-300 from 434176 to 1110016 tid 2 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: writing mira0435601-300 from 1110016 to 1720320 tid 3 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: writing mira0435601-300 from 1720320 to 2039808 tid 4 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 133 current snap is 0 2014-07-01 16:33:56,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: seq_num 1037 ranges {430080=507904,937984=614400,1552384=434176,1986560=155648} 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: writing mira0435601-133 from 430080 to 937984 tid 1 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: writing mira0435601-133 from 937984 to 1552384 tid 2 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: writing mira0435601-133 from 1552384 to 1986560 tid 3 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: writing mira0435601-133 from 1986560 to 2142208 tid 4 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 412 current snap is 0 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: seq_num 1038 ranges {0=225280} 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: writing mira0435601-412 from 0 to 225280 tid 1 2014-07-01 16:33:56,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:33:56,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1070 16 4896 4880 18.2396 0 - 3.44729 2014-07-01 16:33:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 1 to mira0435601-141 2014-07-01 16:33:56,914.914 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:33:57,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1071 16 4899 4883 18.2337 6 2.53548 3.4477 2014-07-01 16:33:57,673.673 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:33:58,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: finishing write tid 1 to mira0435601-25 2014-07-01 16:33:58,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: finishing write tid 2 to mira0435601-25 2014-07-01 16:33:58,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: finishing write tid 4 to mira0435601-25 2014-07-01 16:33:58,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1072 16 4901 4885 18.2242 8 3.95882 3.44791 2014-07-01 16:33:59,236.236 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:33:59,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1073 16 4902 4886 18.2109 4 3.73227 3.44796 2014-07-01 16:33:59,932.932 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'45 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'67 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2867/11118 objects degraded (25.787%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3169827, u'degraded_objects': 2867, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21388214191, u'degraded_total': 11118, u'bytes_used': 38129721344, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 14962782, u'op_per_sec': 7, u'version': 918, u'pgs_by_state': [{u'count': 45, u'state_name': u'active+recovery_wait'}, {u'count': 176, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 484451, u'degraded_ratio': u'25.787', u'bytes_avail': 4960452816896}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 463, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:33:59,932.932 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:00,462.462 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1074 16 4904 4888 18.2014 8 5.31218 3.44872 2014-07-01 16:34:00,663.663 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:34:00,663.663 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:00,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: expect (ObjNum 15 snap 0 seq_num 15) 2014-07-01 16:34:01,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: done (15 left) 2014-07-01 16:34:01,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 16:34:01,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: seq_num 1039 ranges {2056192=655360,2711552=425984,3137536=376832} 2014-07-01 16:34:01,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: writing mira0435601-311 from 2056192 to 2711552 tid 1 2014-07-01 16:34:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: writing mira0435601-311 from 2711552 to 3137536 tid 2 2014-07-01 16:34:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: writing mira0435601-311 from 3137536 to 3514368 tid 3 2014-07-01 16:34:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:34:01,427.427 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:01,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1075 16 4905 4889 18.1882 4 6.32796 3.44931 2014-07-01 16:34:01,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 1 to mira0435601-300 2014-07-01 16:34:01,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 2 to mira0435601-300 2014-07-01 16:34:01,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 2 to mira0435601-141 2014-07-01 16:34:01,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 3 to mira0435601-141 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: expect (ObjNum 412 snap 0 seq_num 412) 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 3 to mira0435601-300 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 4 to mira0435601-300 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 6 to mira0435601-300 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 4 to mira0435601-141 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 6 to mira0435601-141 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: finishing write tid 1 to mira0435601-439 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: finishing write tid 3 to mira0435601-439 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 1 to mira0435601-230 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 2 to mira0435601-230 2014-07-01 16:34:01,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 3 to mira0435601-230 2014-07-01 16:34:01,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 5 to mira0435601-230 2014-07-01 16:34:01,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: done (15 left) 2014-07-01 16:34:01,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1492: delete oid 192 current snap is 0 2014-07-01 16:34:01,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: expect (ObjNum 986 snap 0 seq_num 986) 2014-07-01 16:34:02,483.483 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1076 16 4909 4893 18.1862 16 6.98273 3.44928 2014-07-01 16:34:03,463.463 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1077 16 4909 4893 18.1693 0 - 3.44928 2014-07-01 16:34:04,463.463 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1078 16 4909 4893 18.1524 0 - 3.44928 2014-07-01 16:34:05,463.463 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1079 16 4920 4904 18.1764 14.6667 0.726773 3.45347 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: finishing write tid 1 to mira0435601-412 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: finishing write tid 3 to mira0435601-412 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: finishing write tid 1 to mira0435601-311 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: finishing write tid 2 to mira0435601-311 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: finishing write tid 3 to mira0435601-311 2014-07-01 16:34:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: finishing write tid 5 to mira0435601-311 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: finishing write tid 4 to mira0435601-412 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 412 v 176 (ObjNum 1038 snap 0 seq_num 1038) dirty exists 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 7 to mira0435601-141 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 224 (ObjNum 1027 snap 0 seq_num 1027) dirty exists 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: finishing write tid 4 to mira0435601-439 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 177 (ObjNum 1029 snap 0 seq_num 1029) dirty exists 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 6 to mira0435601-230 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 181 (ObjNum 1032 snap 0 seq_num 1032) dirty exists 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: finishing write tid 6 to mira0435601-311 2014-07-01 16:34:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 180 (ObjNum 1039 snap 0 seq_num 1039) dirty exists 2014-07-01 16:34:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: finishing write tid 1 to mira0435601-111 2014-07-01 16:34:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 1 to mira0435601-366 2014-07-01 16:34:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: finishing write tid 5 to mira0435601-25 2014-07-01 16:34:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 223 (ObjNum 1025 snap 0 seq_num 1025) dirty exists 2014-07-01 16:34:05,822.822 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:06,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:34:06.376747min lat: 0.057269 max lat: 68.3165 avg lat: 3.45117 2014-07-01 16:34:06,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:34:06,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1080 16 4924 4908 18.1744 16 0.61243 3.45117 2014-07-01 16:34:06,581.581 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:07,473.473 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1081 16 4924 4908 18.1576 0 - 3.45117 2014-07-01 16:34:08,120.120 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:08,464.464 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1082 16 4924 4908 18.1408 0 - 3.45117 2014-07-01 16:34:08,818.818 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.016824', u'skew': u'-0.006527', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002350', u'skew': u'-0.021381', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:33:11.315562', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886934356, u'health': u'HEALTH_OK', u'kb_used': 9846696, u'store_stats': {u'bytes_total': 4618836, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'45 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'66 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2872/11183 objects degraded (25.682%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 778207, u'degraded_objects': 2872, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21499092814, u'degraded_total': 11183, u'bytes_used': 38318592000, u'recovering_objects_per_sec': 0, u'version': 929, u'pgs_by_state': [{u'count': 45, u'state_name': u'active+recovery_wait'}, {u'count': 177, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'25.682', u'bytes_avail': 4960263946240}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 468, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:08,818.818 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:09,464.464 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1083 16 4924 4908 18.124 0 - 3.45117 2014-07-01 16:34:09,585.585 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:34:09,586.586 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:10,345.345 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:10,464.464 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1084 16 4924 4908 18.1073 0 - 3.45117 2014-07-01 16:34:11,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: finishing write tid 2 to mira0435601-111 2014-07-01 16:34:11,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 2 to mira0435601-366 2014-07-01 16:34:11,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 3 to mira0435601-366 2014-07-01 16:34:11,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: finishing write tid 3 to mira0435601-111 2014-07-01 16:34:11,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 4 to mira0435601-366 2014-07-01 16:34:11,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: finishing write tid 7 to mira0435601-300 2014-07-01 16:34:11,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 174 (ObjNum 1036 snap 0 seq_num 1036) dirty exists 2014-07-01 16:34:11,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: expect (ObjNum 905 snap 0 seq_num 905) 2014-07-01 16:34:11,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1085 16 4929 4913 18.109 4 13.1284 3.46122 2014-07-01 16:34:12,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1086 16 4942 4926 18.1402 52 0.116765 3.46182 2014-07-01 16:34:13,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1087 16 4942 4926 18.1236 0 - 3.46182 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 6 to mira0435601-366 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: finishing write tid 5 to mira0435601-111 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: finishing write tid 7 to mira0435601-366 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 183 (ObjNum 1034 snap 0 seq_num 1034) dirty exists 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: finishing write tid 6 to mira0435601-111 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 123 (ObjNum 1030 snap 0 seq_num 1030) dirty exists 2014-07-01 16:34:13,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: expect (ObjNum 924 snap 0 seq_num 924) 2014-07-01 16:34:14,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1088 16 4945 4929 18.1179 6 2.02552 3.46092 2014-07-01 16:34:14,750.750 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:15,301.301 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:15,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1089 16 4953 4937 18.1307 32 0.341693 3.46545 2014-07-01 16:34:15,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: finishing write tid 1 to mira0435601-279 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: finishing write tid 2 to mira0435601-279 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: finishing write tid 4 to mira0435601-279 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 1 to mira0435601-133 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 2 to mira0435601-133 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 3 to mira0435601-133 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 4 to mira0435601-133 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 6 to mira0435601-133 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: finishing write tid 5 to mira0435601-279 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 157 (ObjNum 1035 snap 0 seq_num 1035) dirty exists 2014-07-01 16:34:15,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: finishing write tid 7 to mira0435601-133 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 162 (ObjNum 1037 snap 0 seq_num 1037) dirty exists 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: done (15 left) 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: done (14 left) 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: done (13 left) 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: done (12 left) 2014-07-01 16:34:15,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: done (11 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: done (10 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: done (9 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: done (8 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: done (7 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: done (6 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: done (5 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: done (4 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: done (3 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: done (2 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1492: done (1 left) 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 16:34:15,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: seq_num 1041 ranges {5419008=798720,6217728=401408} 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: writing mira0435601-250 from 5419008 to 6217728 tid 1 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: writing mira0435601-250 from 6217728 to 6619136 tid 2 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1494: read oid 328 snap -1 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: read oid 88 snap -1 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: read oid 229 snap -1 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: read oid 314 snap -1 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 258 current snap is 0 2014-07-01 16:34:15,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: seq_num 1042 ranges {3174400=475136,3649536=401408} 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: writing mira0435601-258 from 3174400 to 3649536 tid 1 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: writing mira0435601-258 from 3649536 to 4050944 tid 2 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1499: read oid 224 snap -1 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: seq_num 1043 ranges {1966080=499712,2465792=651264,3117056=430080,3547136=524288} 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: writing mira0435601-445 from 1966080 to 2465792 tid 1 2014-07-01 16:34:15,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: writing mira0435601-445 from 2465792 to 3117056 tid 2 2014-07-01 16:34:15,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: writing mira0435601-445 from 3117056 to 3547136 tid 3 2014-07-01 16:34:15,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: writing mira0435601-445 from 3547136 to 4071424 tid 4 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: seq_num 1044 ranges {2760704=618496} 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: writing mira0435601-359 from 2760704 to 3379200 tid 1 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 464 current snap is 0 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: seq_num 1045 ranges {3858432=647168,4505600=417792,4923392=729088,5652480=57344} 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: writing mira0435601-464 from 3858432 to 4505600 tid 1 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: writing mira0435601-464 from 4505600 to 4923392 tid 2 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: writing mira0435601-464 from 4923392 to 5652480 tid 3 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: writing mira0435601-464 from 5652480 to 5709824 tid 4 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: read oid 278 snap -1 2014-07-01 16:34:16,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: expect (ObjNum 662 snap 0 seq_num 662) 2014-07-01 16:34:16,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1090 16 4953 4937 18.114 0 - 3.46545 2014-07-01 16:34:17,034.034 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:17,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1091 16 4953 4937 18.0974 0 - 3.46545 2014-07-01 16:34:17,747.747 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'44 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2872/11248 objects degraded (25.533%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 35819871, u'degraded_objects': 2872, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21628931778, u'degraded_total': 11248, u'bytes_used': 38566416384, u'recovering_objects_per_sec': 8, u'write_bytes_sec': 23887687, u'op_per_sec': 34, u'version': 940, u'pgs_by_state': [{u'count': 44, u'state_name': u'active+recovery_wait'}, {u'count': 178, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 7562555, u'degraded_ratio': u'25.533', u'bytes_avail': 4960016121856}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 474, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:17,747.747 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:18,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1092 16 4953 4937 18.0809 0 - 3.46545 2014-07-01 16:34:18,483.483 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:34:18,483.483 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:19,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: finishing write tid 1 to mira0435601-258 2014-07-01 16:34:19,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: finishing write tid 2 to mira0435601-258 2014-07-01 16:34:19,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: finishing write tid 4 to mira0435601-258 2014-07-01 16:34:19,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: finishing write tid 5 to mira0435601-258 2014-07-01 16:34:19,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 258 v 221 (ObjNum 1042 snap 0 seq_num 1042) dirty exists 2014-07-01 16:34:19,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: done (11 left) 2014-07-01 16:34:19,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: done (10 left) 2014-07-01 16:34:19,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 16 current snap is 0 2014-07-01 16:34:19,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: seq_num 1046 ranges {167936=720896,888832=647168,1536000=155648} 2014-07-01 16:34:19,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: writing mira0435601-16 from 167936 to 888832 tid 1 2014-07-01 16:34:19,254.254 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:19,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: writing mira0435601-16 from 888832 to 1536000 tid 2 2014-07-01 16:34:19,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: writing mira0435601-16 from 1536000 to 1691648 tid 3 2014-07-01 16:34:19,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1505: delete oid 108 current snap is 0 2014-07-01 16:34:19,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 1 to mira0435601-464 2014-07-01 16:34:19,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1494: expect (ObjNum 604 snap 0 seq_num 604) 2014-07-01 16:34:19,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1093 16 4953 4937 18.0643 0 - 3.46545 2014-07-01 16:34:20,465.465 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1094 16 4953 4937 18.0478 0 - 3.46545 2014-07-01 16:34:21,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1095 16 4954 4938 18.035 0.666667 294.525 3.52439 2014-07-01 16:34:22,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1096 16 4954 4938 18.0185 0 - 3.52439 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 2 to mira0435601-464 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 3 to mira0435601-464 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 4 to mira0435601-464 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 6 to mira0435601-464 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: finishing write tid 7 to mira0435601-464 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 281 (ObjNum 1045 snap 0 seq_num 1045) dirty exists 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: finishing write tid 1 to mira0435601-16 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: finishing write tid 2 to mira0435601-16 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: finishing write tid 3 to mira0435601-16 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: finishing write tid 5 to mira0435601-16 2014-07-01 16:34:23,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: finishing write tid 6 to mira0435601-16 2014-07-01 16:34:23,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 150 (ObjNum 1046 snap 0 seq_num 1046) dirty exists 2014-07-01 16:34:23,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1097 16 4954 4938 18.0021 0 - 3.52439 2014-07-01 16:34:23,762.762 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:23,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: finishing write tid 1 to mira0435601-496 2014-07-01 16:34:23,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: finishing write tid 2 to mira0435601-496 2014-07-01 16:34:23,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: finishing write tid 4 to mira0435601-496 2014-07-01 16:34:23,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: finishing write tid 5 to mira0435601-496 2014-07-01 16:34:23,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 122 (ObjNum 1033 snap 0 seq_num 1033) dirty exists 2014-07-01 16:34:23,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: expect (ObjNum 546 snap 0 seq_num 546) 2014-07-01 16:34:24,352.352 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:24,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1098 16 4969 4953 18.0403 20 0.326171 3.5356 2014-07-01 16:34:25,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: expect (ObjNum 658 snap 0 seq_num 658) 2014-07-01 16:34:25,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1499: expect (ObjNum 223 snap 0 seq_num 223) 2014-07-01 16:34:25,466.466 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1099 16 4970 4954 18.0276 4 0.511384 3.53499 2014-07-01 16:34:25,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: expect (ObjNum 901 snap 0 seq_num 901) 2014-07-01 16:34:26,253.253 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:26,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:34:26.380054min lat: 0.057269 max lat: 294.525 avg lat: 3.53427 2014-07-01 16:34:26,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:34:26,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1100 16 4972 4956 18.0184 8 1.76329 3.53427 2014-07-01 16:34:26,988.988 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2867/11274 objects degraded (25.430%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2867, u'num_pgs': 244, u'data_bytes': 21688774278, u'degraded_total': 11274, u'bytes_used': 38588280832, u'version': 951, u'pgs_by_state': [{u'count': 43, u'state_name': u'active+recovery_wait'}, {u'count': 178, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'25.430', u'bytes_avail': 4959994257408}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 480, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:26,989.989 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:27,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1101 16 4972 4956 18.0021 0 - 3.53427 2014-07-01 16:34:27,775.775 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:34:27,776.776 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:28,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1102 16 4978 4962 18.0075 12 0.38942 3.532 2014-07-01 16:34:28,514.514 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:28,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 1 to mira0435601-359 2014-07-01 16:34:28,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 3 to mira0435601-359 2014-07-01 16:34:29,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 4 to mira0435601-359 2014-07-01 16:34:29,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 165 (ObjNum 1044 snap 0 seq_num 1044) dirty exists 2014-07-01 16:34:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: done (11 left) 2014-07-01 16:34:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1494: done (10 left) 2014-07-01 16:34:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: done (9 left) 2014-07-01 16:34:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: done (8 left) 2014-07-01 16:34:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1499: done (7 left) 2014-07-01 16:34:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: done (6 left) 2014-07-01 16:34:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: done (5 left) 2014-07-01 16:34:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: done (4 left) 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: done (3 left) 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1505: done (2 left) 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1506: read oid 152 snap -1 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: seq_num 1048 ranges {4169728=466944,4636672=462848,5099520=339968} 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: writing mira0435601-21 from 4169728 to 4636672 tid 1 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: writing mira0435601-21 from 4636672 to 5099520 tid 2 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: writing mira0435601-21 from 5099520 to 5439488 tid 3 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 493 current snap is 0 2014-07-01 16:34:29,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: seq_num 1049 ranges {0=90112} 2014-07-01 16:34:29,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: writing mira0435601-493 from 0 to 90112 tid 1 2014-07-01 16:34:29,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1509: read oid 362 snap -1 2014-07-01 16:34:29,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1506: done (5 left) 2014-07-01 16:34:29,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1510: read oid 215 snap -1 2014-07-01 16:34:29,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1511: delete oid 108 current snap is 0 2014-07-01 16:34:29,467.467 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1103 16 4991 4975 18.0383 52 5.27395 3.54533 2014-07-01 16:34:29,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1511: done (6 left) 2014-07-01 16:34:29,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:1512: read oid 269 snap -1 2014-07-01 16:34:29,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: read oid 133 snap -1 2014-07-01 16:34:29,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 16:34:29,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: seq_num 1051 ranges {3952640=688128} 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: writing mira0435601-252 from 3952640 to 4640768 tid 1 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 80 current snap is 0 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: seq_num 1052 ranges {1437696=712704,2150400=442368,2592768=548864,3141632=212992} 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: writing mira0435601-80 from 1437696 to 2150400 tid 1 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: writing mira0435601-80 from 2150400 to 2592768 tid 2 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: writing mira0435601-80 from 2592768 to 3141632 tid 3 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: writing mira0435601-80 from 3141632 to 3354624 tid 4 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 412 current snap is 0 2014-07-01 16:34:29,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: seq_num 1053 ranges {225280=585728,811008=782336,1593344=421888} 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: writing mira0435601-412 from 225280 to 811008 tid 1 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: writing mira0435601-412 from 811008 to 1593344 tid 2 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: writing mira0435601-412 from 1593344 to 2015232 tid 3 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 310 current snap is 0 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: seq_num 1054 ranges {1921024=454656,2375680=454656,2830336=495616,3325952=253952} 2014-07-01 16:34:29,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: writing mira0435601-310 from 1921024 to 2375680 tid 1 2014-07-01 16:34:29,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: writing mira0435601-310 from 2375680 to 2830336 tid 2 2014-07-01 16:34:29,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: writing mira0435601-310 from 2830336 to 3325952 tid 3 2014-07-01 16:34:29,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: writing mira0435601-310 from 3325952 to 3579904 tid 4 2014-07-01 16:34:29,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:1518: read oid 62 snap -1 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1512: done (12 left) 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 167 current snap is 0 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: seq_num 1055 ranges {1490944=778240,2269184=438272,2707456=724992,3432448=442368} 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: writing mira0435601-167 from 1490944 to 2269184 tid 1 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: writing mira0435601-167 from 2269184 to 2707456 tid 2 2014-07-01 16:34:29,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: writing mira0435601-167 from 2707456 to 3432448 tid 3 2014-07-01 16:34:29,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: writing mira0435601-167 from 3432448 to 3874816 tid 4 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: read oid 260 snap -1 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: seq_num 1056 ranges {2179072=405504,2584576=671744,3256320=131072} 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: writing mira0435601-418 from 2179072 to 2584576 tid 1 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: writing mira0435601-418 from 2584576 to 3256320 tid 2 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: writing mira0435601-418 from 3256320 to 3387392 tid 3 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1522: read oid 91 snap -1 2014-07-01 16:34:29,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:34:30,468.468 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1104 16 4994 4978 18.0329 12 0.806548 3.54383 2014-07-01 16:34:31,468.468 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1105 16 4994 4978 18.0165 0 - 3.54383 2014-07-01 16:34:32,468.468 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1106 16 4994 4978 18.0002 0 - 3.54383 2014-07-01 16:34:32,916.916 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:33,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1107 16 4994 4978 17.984 0 - 3.54383 2014-07-01 16:34:33,639.639 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:34,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1108 16 4994 4978 17.9678 0 - 3.54383 2014-07-01 16:34:35,176.176 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:35,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1109 16 4994 4978 17.9515 0 - 3.54383 2014-07-01 16:34:35,930.930 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:33:41.688917', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883549408, u'health': u'HEALTH_OK', u'kb_used': 13231644, u'store_stats': {u'bytes_total': 6716390, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2843/11306 objects degraded (25.146%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 4115480, u'degraded_objects': 2843, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 21755883062, u'degraded_total': 11306, u'bytes_used': 38863720448, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 4115480, u'op_per_sec': 1, u'version': 957, u'pgs_by_state': [{u'count': 43, u'state_name': u'active+recovery_wait'}, {u'count': 178, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 143, u'degraded_ratio': u'25.146', u'bytes_avail': 4959718817792}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 483, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:35,930.930 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:36,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: finishing write tid 1 to mira0435601-252 2014-07-01 16:34:36,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: finishing write tid 1 to mira0435601-493 2014-07-01 16:34:36,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: finishing write tid 3 to mira0435601-493 2014-07-01 16:34:36,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: finishing write tid 3 to mira0435601-252 2014-07-01 16:34:36,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 1 to mira0435601-80 2014-07-01 16:34:36,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 2 to mira0435601-80 2014-07-01 16:34:36,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 3 to mira0435601-80 2014-07-01 16:34:36,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 4 to mira0435601-80 2014-07-01 16:34:36,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 6 to mira0435601-80 2014-07-01 16:34:36,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:1509: expect (ObjNum 948 snap 0 seq_num 948) 2014-07-01 16:34:36,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1110 16 4999 4983 17.9534 3.33333 7.67652 3.54765 2014-07-01 16:34:36,635.635 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:34:36,635.635 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:37,387.387 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:37,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1111 16 5002 4986 17.948 12 1.2038 3.54886 2014-07-01 16:34:37,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: finishing write tid 4 to mira0435601-252 2014-07-01 16:34:37,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 283 (ObjNum 1051 snap 0 seq_num 1051) dirty exists 2014-07-01 16:34:37,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: finishing write tid 7 to mira0435601-80 2014-07-01 16:34:37,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 285 (ObjNum 1052 snap 0 seq_num 1052) dirty exists 2014-07-01 16:34:37,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: finishing write tid 1 to mira0435601-412 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: finishing write tid 2 to mira0435601-412 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: finishing write tid 3 to mira0435601-412 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: finishing write tid 5 to mira0435601-412 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: finishing write tid 4 to mira0435601-493 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 226 (ObjNum 1049 snap 0 seq_num 1049) dirty exists 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: finishing write tid 6 to mira0435601-412 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 412 v 184 (ObjNum 1053 snap 0 seq_num 1053) dirty exists 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: done (15 left) 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1509: done (14 left) 2014-07-01 16:34:37,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: done (13 left) 2014-07-01 16:34:37,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: done (12 left) 2014-07-01 16:34:37,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: done (11 left) 2014-07-01 16:34:37,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1523: read oid 175 snap -1 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 329 current snap is 0 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: seq_num 1057 ranges {1118208=675840,1794048=401408} 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: writing mira0435601-329 from 1118208 to 1794048 tid 1 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: writing mira0435601-329 from 1794048 to 2195456 tid 2 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 498 current snap is 0 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: seq_num 1058 ranges {2621440=548864,3170304=401408} 2014-07-01 16:34:38,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: writing mira0435601-498 from 2621440 to 3170304 tid 1 2014-07-01 16:34:38,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: writing mira0435601-498 from 3170304 to 3571712 tid 2 2014-07-01 16:34:38,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:1526: read oid 128 snap -1 2014-07-01 16:34:38,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:1527: read oid 294 snap -1 2014-07-01 16:34:38,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:34:38,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1523: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 16:34:38,469.469 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1112 16 5004 4988 17.9391 8 8.03928 3.54911 2014-07-01 16:34:39,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1526: expect (ObjNum 734 snap 0 seq_num 734) 2014-07-01 16:34:39,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1113 16 5006 4990 17.9302 8 0.737101 3.54825 2014-07-01 16:34:40,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1114 16 5016 5000 17.95 40 3.8992 3.55892 2014-07-01 16:34:41,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1115 16 5021 5005 17.9518 20 2.48872 3.55747 2014-07-01 16:34:41,785.785 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:42,376.376 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:42,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1116 16 5021 5005 17.9357 0 - 3.55747 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 1 to mira0435601-21 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 2 to mira0435601-21 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 3 to mira0435601-21 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 5 to mira0435601-21 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 6 to mira0435601-21 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 161 (ObjNum 1048 snap 0 seq_num 1048) dirty exists 2014-07-01 16:34:43,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: expect (ObjNum 1037 snap 0 seq_num 1037) 2014-07-01 16:34:43,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1117 16 5021 5005 17.9197 0 - 3.55747 2014-07-01 16:34:44,103.103 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:44,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1118 16 5022 5006 17.9072 1.33333 14.6955 3.55969 2014-07-01 16:34:44,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 1 to mira0435601-498 2014-07-01 16:34:44,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 2 to mira0435601-498 2014-07-01 16:34:44,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1510: expect (ObjNum 844 snap 0 seq_num 844) 2014-07-01 16:34:44,809.809 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:33:54.532862', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886897676, u'health': u'HEALTH_OK', u'kb_used': 9883376, u'store_stats': {u'bytes_total': 8813128, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'10 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2838/11402 objects degraded (24.890%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2838, u'num_pgs': 244, u'data_bytes': 21966413246, u'degraded_total': 11402, u'bytes_used': 39320223744, u'op_per_sec': 0, u'write_bytes_sec': 4178095, u'version': 968, u'pgs_by_state': [{u'count': 43, u'state_name': u'active+recovery_wait'}, {u'count': 178, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'24.890', u'bytes_avail': 4959262314496}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 488, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:44,809.809 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:45,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1119 16 5028 5012 17.9126 24 0.202796 3.56032 2014-07-01 16:34:45,530.530 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:34:45,530.530 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:46,334.334 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:46,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:34:46.383503min lat: 0.057269 max lat: 294.525 avg lat: 3.56194 2014-07-01 16:34:46,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:34:46,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1120 16 5032 5016 17.9109 16 5.05379 3.56194 2014-07-01 16:34:47,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1522: expect (ObjNum 90 snap 0 seq_num 90) 2014-07-01 16:34:47,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1121 16 5034 5018 17.9021 8 1.63527 3.56114 2014-07-01 16:34:48,470.470 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1122 16 5034 5018 17.8861 0 - 3.56114 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 1 to mira0435601-310 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 2 to mira0435601-310 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 3 to mira0435601-310 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 4 to mira0435601-310 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 6 to mira0435601-310 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: finishing write tid 1 to mira0435601-418 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: finishing write tid 2 to mira0435601-418 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: finishing write tid 3 to mira0435601-418 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: finishing write tid 5 to mira0435601-418 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 1 to mira0435601-445 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 2 to mira0435601-445 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 4 to mira0435601-498 2014-07-01 16:34:48,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1527: expect (ObjNum 714 snap 0 seq_num 714) 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 1 to mira0435601-167 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 2 to mira0435601-167 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 3 to mira0435601-167 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 4 to mira0435601-167 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 6 to mira0435601-167 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: done (15 left) 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1510: done (14 left) 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: done (13 left) 2014-07-01 16:34:48,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1522: done (12 left) 2014-07-01 16:34:48,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1523: done (11 left) 2014-07-01 16:34:48,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1526: done (10 left) 2014-07-01 16:34:48,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1527: done (9 left) 2014-07-01 16:34:48,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1528: read oid 300 snap -1 2014-07-01 16:34:49,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1529: delete oid 146 current snap is 0 2014-07-01 16:34:49,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1123 16 5034 5018 17.8702 0 - 3.56114 2014-07-01 16:34:49,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 16:34:50,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1124 16 5049 5033 17.9077 20 3.80015 3.57019 2014-07-01 16:34:50,733.733 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:51,472.472 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1125 16 5050 5034 17.8953 4 1.66244 3.56981 2014-07-01 16:34:51,483.483 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:51,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:1518: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:34:52,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1126 16 5055 5039 17.8972 20 6.01108 3.57003 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 3 to mira0435601-445 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 4 to mira0435601-445 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 6 to mira0435601-445 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: finishing write tid 6 to mira0435601-418 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 193 (ObjNum 1056 snap 0 seq_num 1056) dirty exists 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 5 to mira0435601-498 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 201 (ObjNum 1058 snap 0 seq_num 1058) dirty exists 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: finishing write tid 7 to mira0435601-310 2014-07-01 16:34:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 189 (ObjNum 1054 snap 0 seq_num 1054) dirty exists 2014-07-01 16:34:52,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: finishing write tid 7 to mira0435601-167 2014-07-01 16:34:52,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 167 v 103 (ObjNum 1055 snap 0 seq_num 1055) dirty exists 2014-07-01 16:34:52,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1528: expect (ObjNum 1036 snap 0 seq_num 1036) 2014-07-01 16:34:53,022.022 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:53,471.471 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1127 16 5055 5039 17.8813 0 - 3.57003 2014-07-01 16:34:54,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: finishing write tid 1 to mira0435601-329 2014-07-01 16:34:54,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: finishing write tid 2 to mira0435601-329 2014-07-01 16:34:54,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: finishing write tid 4 to mira0435601-329 2014-07-01 16:34:54,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: finishing write tid 7 to mira0435601-445 2014-07-01 16:34:54,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 172 (ObjNum 1043 snap 0 seq_num 1043) dirty exists 2014-07-01 16:34:54,475.475 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1128 16 5055 5039 17.8655 0 - 3.57003 2014-07-01 16:34:55,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1129 16 5055 5039 17.8496 0 - 3.57003 2014-07-01 16:34:55,588.588 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'65 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'11 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2830/11442 objects degraded (24.733%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2830, u'num_pgs': 244, u'data_bytes': 22054751538, u'degraded_total': 11442, u'bytes_used': 39415889920, u'op_per_sec': 15, u'write_bytes_sec': 6283199, u'version': 978, u'pgs_by_state': [{u'count': 43, u'state_name': u'active+recovery_wait'}, {u'count': 178, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 19417, u'degraded_ratio': u'24.733', u'bytes_avail': 4959166648320}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 494, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:34:55,588.588 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:34:56,278.278 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:34:56,278.278 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:34:56,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1130 16 5070 5054 17.8869 15 4.35492 3.57095 2014-07-01 16:34:57,221.221 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:34:57,473.473 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1131 16 5071 5055 17.8746 4 0.162448 3.57027 2014-07-01 16:34:58,473.473 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1132 16 5071 5055 17.8589 0 - 3.57027 2014-07-01 16:34:59,474.474 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1133 16 5071 5055 17.8431 0 - 3.57027 2014-07-01 16:35:00,474.474 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1134 16 5071 5055 17.8274 0 - 3.57027 2014-07-01 16:35:01,474.474 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1135 16 5071 5055 17.8116 0 - 3.57027 2014-07-01 16:35:01,743.743 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:02,399.399 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:02,475.475 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1136 16 5075 5059 17.81 3.2 0.087232 3.57095 2014-07-01 16:35:03,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1137 16 5076 5060 17.7979 4 0.229176 3.57029 2014-07-01 16:35:04,230.230 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:04,474.474 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1138 16 5076 5060 17.7823 0 - 3.57029 2014-07-01 16:35:04,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: finishing write tid 5 to mira0435601-329 2014-07-01 16:35:04,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 329 v 225 (ObjNum 1057 snap 0 seq_num 1057) dirty exists 2014-07-01 16:35:05,025.025 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:34:11.315911', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886884828, u'health': u'HEALTH_OK', u'kb_used': 9896224, u'store_stats': {u'bytes_total': 9861716, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570244}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'41 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'63 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'10 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2769/11488 objects degraded (24.103%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2769, u'num_pgs': 244, u'data_bytes': 22151220470, u'degraded_total': 11488, u'bytes_used': 39705571328, u'version': 988, u'pgs_by_state': [{u'count': 41, u'state_name': u'active+recovery_wait'}, {u'count': 181, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'24.103', u'bytes_avail': 4958876966912}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 499, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:05,025.025 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:05,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1139 16 5085 5069 17.7982 18 9.86909 3.57939 2014-07-01 16:35:05,775.775 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:35:05,776.776 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:06,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:35:06.387477min lat: 0.057269 max lat: 294.525 avg lat: 3.57609 2014-07-01 16:35:06,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:35:06,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1140 16 5090 5074 17.8002 20 0.457967 3.57609 2014-07-01 16:35:06,604.604 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:07,475.475 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1141 16 5090 5074 17.7846 0 - 3.57609 2014-07-01 16:35:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: done (10 left) 2014-07-01 16:35:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: done (9 left) 2014-07-01 16:35:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1518: done (8 left) 2014-07-01 16:35:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: done (7 left) 2014-07-01 16:35:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: done (6 left) 2014-07-01 16:35:08,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: done (5 left) 2014-07-01 16:35:08,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: done (4 left) 2014-07-01 16:35:08,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: done (3 left) 2014-07-01 16:35:08,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1528: done (2 left) 2014-07-01 16:35:08,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1529: done (1 left) 2014-07-01 16:35:08,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1530: read oid 343 snap -1 2014-07-01 16:35:08,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 193 current snap is 0 2014-07-01 16:35:08,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: seq_num 1060 ranges {1380352=651264,2031616=651264,2682880=552960} 2014-07-01 16:35:08,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0435601-193 from 1380352 to 2031616 tid 1 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0435601-193 from 2031616 to 2682880 tid 2 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0435601-193 from 2682880 to 3235840 tid 3 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: read oid 186 snap -1 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: seq_num 1061 ranges {3407872=368640} 2014-07-01 16:35:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: writing mira0435601-226 from 3407872 to 3776512 tid 1 2014-07-01 16:35:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1534: read oid 291 snap -1 2014-07-01 16:35:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1530: expect (ObjNum 798 snap 0 seq_num 798) 2014-07-01 16:35:08,489.489 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1142 16 5102 5086 17.811 24 0.116088 3.58984 2014-07-01 16:35:09,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1143 16 5109 5093 17.8199 28 1.06931 3.58597 2014-07-01 16:35:10,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1144 16 5115 5099 17.8253 24 0.111865 3.58192 2014-07-01 16:35:11,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: finishing write tid 1 to mira0435601-226 2014-07-01 16:35:11,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: finishing write tid 3 to mira0435601-226 2014-07-01 16:35:11,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: finishing write tid 4 to mira0435601-226 2014-07-01 16:35:11,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 102 (ObjNum 1061 snap 0 seq_num 1061) dirty exists 2014-07-01 16:35:11,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1530: done (5 left) 2014-07-01 16:35:11,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: done (4 left) 2014-07-01 16:35:11,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: delete oid 336 current snap is 0 2014-07-01 16:35:11,427.427 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:11,475.475 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1145 16 5115 5099 17.8098 0 - 3.58192 2014-07-01 16:35:12,170.170 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:12,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1146 16 5115 5099 17.7942 0 - 3.58192 2014-07-01 16:35:13,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1147 16 5115 5099 17.7787 0 - 3.58192 2014-07-01 16:35:13,704.704 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:13,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: finishing write tid 1 to mira0435601-250 2014-07-01 16:35:13,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: finishing write tid 2 to mira0435601-250 2014-07-01 16:35:13,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: finishing write tid 4 to mira0435601-250 2014-07-01 16:35:14,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:1534: expect (ObjNum 290 snap 0 seq_num 290) 2014-07-01 16:35:14,444.444 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2715/11564 objects degraded (23.478%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2715, u'num_pgs': 244, u'data_bytes': 22312828010, u'degraded_total': 11564, u'bytes_used': 39925878784, u'op_per_sec': 15, u'write_bytes_sec': 24432165, u'version': 999, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2017924, u'degraded_ratio': u'23.478', u'bytes_avail': 4958656659456}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 505, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:14,445.445 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:14,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1148 16 5129 5113 17.812 14 4.48455 3.58729 2014-07-01 16:35:15,180.180 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:35:15,180.180 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:15,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: expect (ObjNum 185 snap 0 seq_num 185) 2014-07-01 16:35:15,476.476 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1149 16 5135 5119 17.8174 24 0.229153 3.58642 2014-07-01 16:35:15,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 1 to mira0435601-193 2014-07-01 16:35:15,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 2 to mira0435601-193 2014-07-01 16:35:15,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 3 to mira0435601-193 2014-07-01 16:35:15,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 5 to mira0435601-193 2014-07-01 16:35:15,939.939 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:16,477.477 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1150 16 5135 5119 17.8019 0 - 3.58642 2014-07-01 16:35:17,478.478 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1151 16 5135 5119 17.7864 0 - 3.58642 2014-07-01 16:35:18,478.478 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1152 16 5135 5119 17.771 0 - 3.58642 2014-07-01 16:35:19,478.478 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1153 16 5137 5121 17.7625 2 0.110116 3.58579 2014-07-01 16:35:20,312.312 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:20,479.479 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1154 16 5137 5121 17.7471 0 - 3.58579 2014-07-01 16:35:21,044.044 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:21,479.479 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1155 16 5137 5121 17.7317 0 - 3.58579 2014-07-01 16:35:21,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 6 to mira0435601-193 2014-07-01 16:35:21,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 229 (ObjNum 1060 snap 0 seq_num 1060) dirty exists 2014-07-01 16:35:22,479.479 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1156 16 5147 5131 17.751 13.3333 0.175218 3.59127 2014-07-01 16:35:22,595.595 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:23,284.284 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2730/11641 objects degraded (23.452%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12145304, u'degraded_objects': 2730, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 22480886830, u'degraded_total': 11641, u'bytes_used': 40363171840, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 32393409, u'op_per_sec': 25, u'version': 1008, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 23721, u'degraded_ratio': u'23.452', u'bytes_avail': 4958219366400}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 508, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:23,284.284 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:23,478.478 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1157 16 5147 5131 17.7357 0 - 3.59127 2014-07-01 16:35:24,060.060 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:35:24,060.060 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:24,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: finishing write tid 5 to mira0435601-250 2014-07-01 16:35:24,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 109 (ObjNum 1041 snap 0 seq_num 1041) dirty exists 2014-07-01 16:35:24,479.479 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1158 16 5151 5135 17.7342 8 9.39965 3.59605 2014-07-01 16:35:24,812.812 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:25,479.479 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1159 16 5151 5135 17.7189 0 - 3.59605 2014-07-01 16:35:26,480.480 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:35:26.391697min lat: 0.057269 max lat: 294.525 avg lat: 3.59605 2014-07-01 16:35:26,480.480 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:35:26,481.481 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1160 16 5151 5135 17.7036 0 - 3.59605 2014-07-01 16:35:27,480.480 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1161 16 5153 5137 17.6952 2.66667 0.155659 3.59528 2014-07-01 16:35:28,481.481 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1162 16 5153 5137 17.68 0 - 3.59528 2014-07-01 16:35:29,354.354 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:29,480.480 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1163 16 5153 5137 17.6648 0 - 3.59528 2014-07-01 16:35:30,124.124 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:30,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1164 16 5153 5137 17.6496 0 - 3.59528 2014-07-01 16:35:31,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1165 16 5153 5137 17.6345 0 - 3.59528 2014-07-01 16:35:31,686.686 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:32,392.392 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2728/11661 objects degraded (23.394%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2728, u'num_pgs': 244, u'data_bytes': 22522829710, u'degraded_total': 11661, u'bytes_used': 40481148928, u'op_per_sec': 30, u'write_bytes_sec': 4139699, u'version': 1022, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 40328, u'degraded_ratio': u'23.394', u'bytes_avail': 4958101389312}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 515, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:32,393.393 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:32,481.481 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1166 16 5173 5157 17.6879 16 0.384675 3.60712 2014-07-01 16:35:33,126.126 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:35:33,127.127 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:33,481.481 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1167 16 5181 5165 17.7002 32 0.140418 3.60186 2014-07-01 16:35:33,897.897 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:34,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1168 16 5181 5165 17.685 0 - 3.60186 2014-07-01 16:35:35,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1169 16 5181 5165 17.6699 0 - 3.60186 2014-07-01 16:35:36,483.483 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1170 16 5181 5165 17.6548 0 - 3.60186 2014-07-01 16:35:37,483.483 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1171 16 5186 5170 17.6568 5 0.15587 3.61124 2014-07-01 16:35:38,438.438 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:38,483.483 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1172 16 5198 5182 17.6827 48 0.296477 3.60957 2014-07-01 16:35:39,021.021 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:39,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1173 16 5212 5196 17.7153 56 0.281268 3.61097 2014-07-01 16:35:40,483.483 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1174 16 5220 5204 17.7275 32 0.451468 3.60645 2014-07-01 16:35:40,748.748 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:41,463.463 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:34:41.689360', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883449660, u'health': u'HEALTH_OK', u'kb_used': 13331392, u'store_stats': {u'bytes_total': 9862118, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570646}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2733/11721 objects degraded (23.317%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2733, u'num_pgs': 244, u'data_bytes': 22648658770, u'degraded_total': 11721, u'bytes_used': 40798998528, u'version': 1034, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'23.317', u'bytes_avail': 4957783539712}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 520, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:41,464.464 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:41,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1175 16 5220 5204 17.7124 0 - 3.60645 2014-07-01 16:35:42,225.225 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:35:42,226.226 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:42,482.482 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1176 16 5227 5211 17.7212 14 0.694943 3.60302 2014-07-01 16:35:42,990.990 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:43,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1177 16 5233 5217 17.7265 24 4.27904 3.60336 2014-07-01 16:35:44,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1178 16 5238 5222 17.7284 20 0.640082 3.60199 2014-07-01 16:35:45,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1179 16 5239 5223 17.7168 4 0.133145 3.60133 2014-07-01 16:35:46,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:35:46.396076min lat: 0.057269 max lat: 294.525 avg lat: 3.60133 2014-07-01 16:35:46,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:35:46,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1180 16 5239 5223 17.7018 0 - 3.60133 2014-07-01 16:35:47,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1181 16 5239 5223 17.6868 0 - 3.60133 2014-07-01 16:35:47,533.533 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:48,070.070 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:48,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1182 16 5239 5223 17.6718 0 - 3.60133 2014-07-01 16:35:49,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: done (4 left) 2014-07-01 16:35:49,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: done (3 left) 2014-07-01 16:35:49,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: done (2 left) 2014-07-01 16:35:49,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1534: done (1 left) 2014-07-01 16:35:49,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: done (0 left) 2014-07-01 16:35:49,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 173 current snap is 0 2014-07-01 16:35:49,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: seq_num 1063 ranges {1683456=421888} 2014-07-01 16:35:49,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: writing mira0435601-173 from 1683456 to 2105344 tid 1 2014-07-01 16:35:49,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: delete oid 445 current snap is 0 2014-07-01 16:35:49,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1183 16 5249 5233 17.6907 10 5.49694 3.60221 2014-07-01 16:35:49,797.797 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:50,485.485 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1184 16 5250 5234 17.6791 4 10.2884 3.60348 2014-07-01 16:35:50,537.537 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:34:54.533296', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886840572, u'health': u'HEALTH_OK', u'kb_used': 9940480, u'store_stats': {u'bytes_total': 14056008, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3570232}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2748/11833 objects degraded (23.223%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 8350474, u'degraded_objects': 2748, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 22883539714, u'degraded_total': 11833, u'bytes_used': 41471897600, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 36186067, u'op_per_sec': 12, u'version': 1041, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3397, u'degraded_ratio': u'23.223', u'bytes_avail': 4957110640640}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 523, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:35:50,537.537 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:35:51,316.316 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:35:51,316.316 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:51,484.484 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1185 16 5256 5240 17.6844 24 0.54621 3.6016 2014-07-01 16:35:52,135.135 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:52,516.516 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1186 16 5258 5242 17.6763 8 0.19272 3.60046 2014-07-01 16:35:53,509.509 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1187 16 5258 5242 17.6614 0 - 3.60046 2014-07-01 16:35:54,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1188 16 5258 5242 17.6465 0 - 3.60046 2014-07-01 16:35:55,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1189 16 5267 5251 17.6619 12 10.708 3.61501 2014-07-01 16:35:56,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1190 16 5268 5252 17.6505 4 0.887496 3.61449 2014-07-01 16:35:56,654.654 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:35:57,373.373 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:35:57,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1191 16 5271 5255 17.6457 12 0.832853 3.61301 2014-07-01 16:35:58,486.486 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1192 16 5276 5260 17.6477 20 0.130527 3.61027 2014-07-01 16:35:59,644.644 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1193 16 5276 5260 17.6329 0 - 3.61027 2014-07-01 16:35:59,711.711 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:00,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: done (1 left) 2014-07-01 16:36:00,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: read oid 329 snap -1 2014-07-01 16:36:00,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1539: read oid 46 snap -1 2014-07-01 16:36:00,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 425 current snap is 0 2014-07-01 16:36:00,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: seq_num 1065 ranges {1830912=618496,2449408=401408} 2014-07-01 16:36:00,408.408 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2749/11887 objects degraded (23.126%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2749, u'num_pgs': 244, u'data_bytes': 22996785782, u'degraded_total': 11887, u'bytes_used': 41713303552, u'op_per_sec': 0, u'write_bytes_sec': 4160268, u'version': 1054, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'23.126', u'bytes_avail': 4956869234688}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 530, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:00,408.408 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:00,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: writing mira0435601-425 from 1830912 to 2449408 tid 1 2014-07-01 16:36:00,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: writing mira0435601-425 from 2449408 to 2850816 tid 2 2014-07-01 16:36:00,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:1541: delete oid 466 current snap is 0 2014-07-01 16:36:00,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: expect (ObjNum 1057 snap 0 seq_num 1057) 2014-07-01 16:36:00,557.557 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1194 16 5289 5273 17.6617 26 5.39551 3.62117 2014-07-01 16:36:01,118.118 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:36:01,118.118 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:01,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1195 16 5295 5279 17.667 24 0.710369 3.61964 2014-07-01 16:36:01,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1539: expect (ObjNum 932 snap 0 seq_num 932) 2014-07-01 16:36:01,934.934 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:02,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1196 16 5303 5287 17.6789 32 1.55434 3.61605 2014-07-01 16:36:03,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1197 16 5303 5287 17.6642 0 - 3.61605 2014-07-01 16:36:04,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1198 16 5304 5288 17.6528 2 3.14477 3.61597 2014-07-01 16:36:05,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1199 16 5307 5291 17.6481 12 4.59538 3.61639 2014-07-01 16:36:06,476.476 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:06,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:36:06.400152min lat: 0.057269 max lat: 294.525 avg lat: 3.61639 2014-07-01 16:36:06,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:36:06,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1200 16 5307 5291 17.6333 0 - 3.61639 2014-07-01 16:36:07,242.242 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:07,487.487 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1201 16 5307 5291 17.6187 0 - 3.61639 2014-07-01 16:36:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: done (4 left) 2014-07-01 16:36:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1539: done (3 left) 2014-07-01 16:36:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1541: done (2 left) 2014-07-01 16:36:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 218 current snap is 0 2014-07-01 16:36:08,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: seq_num 1067 ranges {741376=761856} 2014-07-01 16:36:08,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1202 16 5315 5299 17.6306 10.6667 0.145248 3.6186 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: writing mira0435601-218 from 741376 to 1503232 tid 1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: read oid 183 snap -1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: seq_num 1068 ranges {2580480=790528,3371008=454656,3825664=749568} 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: writing mira0435601-439 from 2580480 to 3371008 tid 1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: writing mira0435601-439 from 3371008 to 3825664 tid 2 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: writing mira0435601-439 from 3825664 to 4575232 tid 3 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1545: read oid 26 snap -1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1546: read oid 9 snap -1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1547: read oid 18 snap -1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1548: read oid 77 snap -1 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 86 current snap is 0 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: seq_num 1069 ranges {1359872=462848,1822720=794624,2617344=409600} 2014-07-01 16:36:08,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: writing mira0435601-86 from 1359872 to 1822720 tid 1 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: writing mira0435601-86 from 1822720 to 2617344 tid 2 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: writing mira0435601-86 from 2617344 to 3026944 tid 3 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1550: read oid 135 snap -1 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1545: done (10 left) 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1551: read oid 305 snap -1 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 72 current snap is 0 2014-07-01 16:36:08,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: seq_num 1070 ranges {2789376=737280,3526656=622592,4149248=180224} 2014-07-01 16:36:08,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: writing mira0435601-72 from 2789376 to 3526656 tid 1 2014-07-01 16:36:08,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: writing mira0435601-72 from 3526656 to 4149248 tid 2 2014-07-01 16:36:08,830.830 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: writing mira0435601-72 from 4149248 to 4329472 tid 3 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 420 current snap is 0 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: seq_num 1071 ranges {2543616=610304,3153920=606208,3760128=196608} 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: writing mira0435601-420 from 2543616 to 3153920 tid 1 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: writing mira0435601-420 from 3153920 to 3760128 tid 2 2014-07-01 16:36:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: writing mira0435601-420 from 3760128 to 3956736 tid 3 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 422 current snap is 0 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: seq_num 1072 ranges {8192=233472} 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: writing mira0435601-422 from 8192 to 241664 tid 1 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: read oid 221 snap -1 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: finishing write tid 1 to mira0435601-439 2014-07-01 16:36:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:1546: expect (ObjNum 1004 snap 0 seq_num 1004) 2014-07-01 16:36:09,489.489 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1203 16 5327 5311 17.6559 48 0.363622 3.61901 2014-07-01 16:36:09,576.576 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:35:11.316174', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886845680, u'health': u'HEALTH_OK', u'kb_used': 9935372, u'store_stats': {u'bytes_total': 4373519, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3324927}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2766/11958 objects degraded (23.131%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2766, u'num_pgs': 244, u'data_bytes': 23158159950, u'degraded_total': 11958, u'bytes_used': 42059165696, u'op_per_sec': 17, u'write_bytes_sec': 42795911, u'version': 1061, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2742816, u'degraded_ratio': u'23.131', u'bytes_avail': 4956523372544}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 532, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:09,576.576 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:10,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: finishing write tid 2 to mira0435601-439 2014-07-01 16:36:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: finishing write tid 3 to mira0435601-439 2014-07-01 16:36:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: finishing write tid 5 to mira0435601-439 2014-07-01 16:36:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: finishing write tid 6 to mira0435601-439 2014-07-01 16:36:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 185 (ObjNum 1068 snap 0 seq_num 1068) dirty exists 2014-07-01 16:36:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: done (14 left) 2014-07-01 16:36:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1546: done (13 left) 2014-07-01 16:36:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1556: delete oid 271 current snap is 0 2014-07-01 16:36:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1548: expect (ObjNum 76 snap 0 seq_num 76) 2014-07-01 16:36:10,300.300 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:36:10,300.300 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:10,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1204 16 5327 5311 17.6412 0 - 3.61901 2014-07-01 16:36:11,050.050 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:11,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1205 16 5327 5311 17.6266 0 - 3.61901 2014-07-01 16:36:11,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: expect (ObjNum 220 snap 0 seq_num 220) 2014-07-01 16:36:12,489.489 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1206 16 5339 5323 17.6517 16 0.135464 3.61393 2014-07-01 16:36:12,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: finishing write tid 1 to mira0435601-420 2014-07-01 16:36:12,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: finishing write tid 2 to mira0435601-420 2014-07-01 16:36:12,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: finishing write tid 3 to mira0435601-420 2014-07-01 16:36:12,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: finishing write tid 5 to mira0435601-420 2014-07-01 16:36:12,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: finishing write tid 1 to mira0435601-72 2014-07-01 16:36:12,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1550: expect (ObjNum 704 snap 0 seq_num 704) 2014-07-01 16:36:13,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1207 16 5343 5327 17.6504 16 0.095115 3.6113 2014-07-01 16:36:14,489.489 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1208 16 5343 5327 17.6358 0 - 3.6113 2014-07-01 16:36:14,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: finishing write tid 2 to mira0435601-72 2014-07-01 16:36:14,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: finishing write tid 3 to mira0435601-72 2014-07-01 16:36:14,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: finishing write tid 5 to mira0435601-72 2014-07-01 16:36:14,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: finishing write tid 6 to mira0435601-72 2014-07-01 16:36:14,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 188 (ObjNum 1070 snap 0 seq_num 1070) dirty exists 2014-07-01 16:36:15,453.453 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:15,488.488 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1209 16 5351 5335 17.6476 16 0.146279 3.61419 2014-07-01 16:36:16,032.032 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:16,489.489 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1210 16 5352 5336 17.6364 4 1.09756 3.61371 2014-07-01 16:36:17,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1211 16 5352 5336 17.6218 0 - 3.61371 2014-07-01 16:36:17,753.753 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:18,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1212 16 5358 5342 17.627 12 0.167282 3.62086 2014-07-01 16:36:18,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 16:36:18,537.537 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2778/12035 objects degraded (23.083%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2778, u'num_pgs': 244, u'data_bytes': 23328643522, u'degraded_total': 12035, u'bytes_used': 42555981824, u'op_per_sec': 58, u'write_bytes_sec': 62799624, u'version': 1072, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3085611, u'degraded_ratio': u'23.083', u'bytes_avail': 4956026556416}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 538, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:18,538.538 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:19,265.265 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:36:19,265.265 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:19,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 1 to mira0435601-173 2014-07-01 16:36:19,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: finishing write tid 6 to mira0435601-420 2014-07-01 16:36:19,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 233 (ObjNum 1071 snap 0 seq_num 1071) dirty exists 2014-07-01 16:36:19,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:1547: expect (ObjNum 584 snap 0 seq_num 584) 2014-07-01 16:36:19,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1213 16 5368 5352 17.6455 40 0.338782 3.62108 2014-07-01 16:36:20,032.032 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:20,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 3 to mira0435601-173 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: finishing write tid 1 to mira0435601-86 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 1 to mira0435601-218 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 3 to mira0435601-218 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: finishing write tid 2 to mira0435601-86 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: finishing write tid 3 to mira0435601-86 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: finishing write tid 5 to mira0435601-86 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: finishing write tid 1 to mira0435601-422 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: finishing write tid 3 to mira0435601-422 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: finishing write tid 6 to mira0435601-86 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 187 (ObjNum 1069 snap 0 seq_num 1069) dirty exists 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: finishing write tid 4 to mira0435601-422 2014-07-01 16:36:20,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 163 (ObjNum 1072 snap 0 seq_num 1072) dirty exists 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: done (13 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1547: done (12 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1548: done (11 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: done (10 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1550: done (9 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1551: done (8 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: done (7 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: done (6 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: done (5 left) 2014-07-01 16:36:20,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: done (4 left) 2014-07-01 16:36:20,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:1556: done (3 left) 2014-07-01 16:36:20,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 368 current snap is 0 2014-07-01 16:36:20,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: seq_num 1074 ranges {2252800=425984,2678784=401408} 2014-07-01 16:36:20,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: writing mira0435601-368 from 2252800 to 2678784 tid 1 2014-07-01 16:36:20,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: writing mira0435601-368 from 2678784 to 3080192 tid 2 2014-07-01 16:36:20,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: delete oid 466 current snap is 0 2014-07-01 16:36:20,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1214 16 5368 5352 17.6309 0 - 3.62108 2014-07-01 16:36:21,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1215 16 5370 5354 17.623 4 2.75545 3.62079 2014-07-01 16:36:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: done (4 left) 2014-07-01 16:36:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 140 current snap is 0 2014-07-01 16:36:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: seq_num 1076 ranges {3284992=729088,4014080=663552,4677632=540672} 2014-07-01 16:36:21,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: writing mira0435601-140 from 3284992 to 4014080 tid 1 2014-07-01 16:36:21,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: writing mira0435601-140 from 4014080 to 4677632 tid 2 2014-07-01 16:36:21,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: writing mira0435601-140 from 4677632 to 5218304 tid 3 2014-07-01 16:36:21,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: read oid 297 snap -1 2014-07-01 16:36:21,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: read oid 167 snap -1 2014-07-01 16:36:21,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: read oid 72 snap -1 2014-07-01 16:36:21,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1563: read oid 331 snap -1 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1564: read oid 131 snap -1 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: read oid 54 snap -1 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1566: read oid 103 snap -1 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 114 current snap is 0 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: seq_num 1077 ranges {937984=598016,1536000=598016,2134016=503808,2637824=102400} 2014-07-01 16:36:21,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: writing mira0435601-114 from 937984 to 1536000 tid 1 2014-07-01 16:36:21,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: writing mira0435601-114 from 1536000 to 2134016 tid 2 2014-07-01 16:36:21,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: writing mira0435601-114 from 2134016 to 2637824 tid 3 2014-07-01 16:36:21,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: writing mira0435601-114 from 2637824 to 2740224 tid 4 2014-07-01 16:36:21,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 165 current snap is 0 2014-07-01 16:36:21,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: seq_num 1078 ranges {1609728=675840,2285568=430080,2715648=372736} 2014-07-01 16:36:21,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: writing mira0435601-165 from 1609728 to 2285568 tid 1 2014-07-01 16:36:21,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: writing mira0435601-165 from 2285568 to 2715648 tid 2 2014-07-01 16:36:21,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: writing mira0435601-165 from 2715648 to 3088384 tid 3 2014-07-01 16:36:21,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:36:21,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: seq_num 1079 ranges {9461760=548864,10010624=413696,10424320=389120} 2014-07-01 16:36:22,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: writing mira0435601-337 from 9461760 to 10010624 tid 1 2014-07-01 16:36:22,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: writing mira0435601-337 from 10010624 to 10424320 tid 2 2014-07-01 16:36:22,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: writing mira0435601-337 from 10424320 to 10813440 tid 3 2014-07-01 16:36:22,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 45 current snap is 0 2014-07-01 16:36:22,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: seq_num 1080 ranges {4763648=417792,5181440=643072,5824512=159744} 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: writing mira0435601-45 from 4763648 to 5181440 tid 1 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: writing mira0435601-45 from 5181440 to 5824512 tid 2 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: writing mira0435601-45 from 5824512 to 5984256 tid 3 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: finishing write tid 1 to mira0435601-337 2014-07-01 16:36:22,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: finishing write tid 2 to mira0435601-337 2014-07-01 16:36:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1563: expect (ObjNum 816 snap 0 seq_num 816) 2014-07-01 16:36:22,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1216 16 5380 5364 17.6414 40 0.272797 3.61607 2014-07-01 16:36:23,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1217 16 5381 5365 17.6302 4 0.455504 3.61548 2014-07-01 16:36:23,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1564: expect (ObjNum 815 snap 0 seq_num 815) 2014-07-01 16:36:24,402.402 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:24,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1218 16 5381 5365 17.6157 0 - 3.61548 2014-07-01 16:36:25,132.132 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:25,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: finishing write tid 3 to mira0435601-337 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: finishing write tid 5 to mira0435601-337 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 1 to mira0435601-114 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 2 to mira0435601-114 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 3 to mira0435601-114 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 4 to mira0435601-114 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 6 to mira0435601-114 2014-07-01 16:36:25,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: finishing write tid 6 to mira0435601-337 2014-07-01 16:36:25,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 287 (ObjNum 1079 snap 0 seq_num 1079) dirty exists 2014-07-01 16:36:25,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: finishing write tid 7 to mira0435601-114 2014-07-01 16:36:25,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 155 (ObjNum 1077 snap 0 seq_num 1077) dirty exists 2014-07-01 16:36:25,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: expect (ObjNum 1070 snap 0 seq_num 1070) 2014-07-01 16:36:25,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1219 16 5381 5365 17.6013 0 - 3.61548 2014-07-01 16:36:26,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:36:26.403539min lat: 0.057269 max lat: 294.525 avg lat: 3.61548 2014-07-01 16:36:26,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:36:26,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1220 16 5381 5365 17.5869 0 - 3.61548 2014-07-01 16:36:26,655.655 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:27,380.380 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2775/12087 objects degraded (22.959%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2775, u'num_pgs': 244, u'data_bytes': 23446702470, u'degraded_total': 12087, u'bytes_used': 42644897792, u'version': 1080, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.959', u'bytes_avail': 4955937640448}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 541, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:27,381.381 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:27,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1221 16 5381 5365 17.5725 0 - 3.61548 2014-07-01 16:36:27,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: finishing write tid 1 to mira0435601-165 2014-07-01 16:36:27,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: finishing write tid 2 to mira0435601-165 2014-07-01 16:36:27,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: done (15 left) 2014-07-01 16:36:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1563: done (14 left) 2014-07-01 16:36:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1564: done (13 left) 2014-07-01 16:36:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: done (12 left) 2014-07-01 16:36:27,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: done (11 left) 2014-07-01 16:36:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: read oid 158 snap -1 2014-07-01 16:36:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: finishing write tid 3 to mira0435601-165 2014-07-01 16:36:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: finishing write tid 5 to mira0435601-165 2014-07-01 16:36:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: finishing write tid 6 to mira0435601-165 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 192 (ObjNum 1078 snap 0 seq_num 1078) dirty exists 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: done (11 left) 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 449 current snap is 0 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: seq_num 1081 ranges {798720=45056} 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: writing mira0435601-449 from 798720 to 843776 tid 1 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 291 current snap is 0 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: seq_num 1082 ranges {1818624=770048} 2014-07-01 16:36:27,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: writing mira0435601-291 from 1818624 to 2588672 tid 1 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: read oid 260 snap -1 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: seq_num 1083 ranges {3043328=794624,3837952=794624,4632576=409600} 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: writing mira0435601-382 from 3043328 to 3837952 tid 1 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: writing mira0435601-382 from 3837952 to 4632576 tid 2 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: writing mira0435601-382 from 4632576 to 5042176 tid 3 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 93 current snap is 0 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: seq_num 1084 ranges {536576=667648,1204224=626688,1830912=577536} 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: writing mira0435601-93 from 536576 to 1204224 tid 1 2014-07-01 16:36:27,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: writing mira0435601-93 from 1204224 to 1830912 tid 2 2014-07-01 16:36:27,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: writing mira0435601-93 from 1830912 to 2408448 tid 3 2014-07-01 16:36:27,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:36:28,118.118 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:36:28,119.119 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:28,490.490 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1222 16 5381 5365 17.5581 0 - 3.61548 2014-07-01 16:36:28,915.915 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:29,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1223 16 5381 5365 17.5437 0 - 3.61548 2014-07-01 16:36:30,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1224 16 5389 5373 17.5555 4.57143 0.151878 3.62169 2014-07-01 16:36:31,491.491 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1225 16 5389 5373 17.5412 0 - 3.62169 2014-07-01 16:36:32,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1226 16 5389 5373 17.5269 0 - 3.62169 2014-07-01 16:36:32,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 4 to mira0435601-173 2014-07-01 16:36:32,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 111 (ObjNum 1063 snap 0 seq_num 1063) dirty exists 2014-07-01 16:36:32,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: done (15 left) 2014-07-01 16:36:32,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:36:32,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: seq_num 1085 ranges {4575232=540672,5115904=561152,5677056=643072} 2014-07-01 16:36:32,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: writing mira0435601-439 from 4575232 to 5115904 tid 1 2014-07-01 16:36:32,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: writing mira0435601-439 from 5115904 to 5677056 tid 2 2014-07-01 16:36:32,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: writing mira0435601-439 from 5677056 to 6320128 tid 3 2014-07-01 16:36:32,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 4 to mira0435601-218 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 167 (ObjNum 1067 snap 0 seq_num 1067) dirty exists 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: done (15 left) 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1578: read oid 380 snap -1 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: finishing write tid 1 to mira0435601-425 2014-07-01 16:36:32,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: finishing write tid 2 to mira0435601-425 2014-07-01 16:36:32,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: finishing write tid 1 to mira0435601-439 2014-07-01 16:36:32,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: expect (ObjNum 296 snap 0 seq_num 296) 2014-07-01 16:36:33,318.318 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:33,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1227 16 5409 5393 17.5778 26.6667 0.241412 3.63613 2014-07-01 16:36:34,088.088 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:34,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1228 16 5415 5399 17.583 24 11.327 3.63616 2014-07-01 16:36:34,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: finishing write tid 4 to mira0435601-425 2014-07-01 16:36:34,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: finishing write tid 5 to mira0435601-425 2014-07-01 16:36:34,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 108 (ObjNum 1065 snap 0 seq_num 1065) dirty exists 2014-07-01 16:36:34,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: expect (ObjNum 1055 snap 0 seq_num 1055) 2014-07-01 16:36:35,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1229 16 5415 5399 17.5687 0 - 3.63616 2014-07-01 16:36:35,612.612 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:36,354.354 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:35:41.689665', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883365192, u'health': u'HEALTH_OK', u'kb_used': 13415860, u'store_stats': {u'bytes_total': 5560872, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'61 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2788/12145 objects degraded (22.956%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 4144122, u'degraded_objects': 2788, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 23574194466, u'degraded_total': 12145, u'bytes_used': 42892042240, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 26588756, u'op_per_sec': 10, u'version': 1092, u'pgs_by_state': [{u'count': 39, u'state_name': u'active+recovery_wait'}, {u'count': 183, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1011, u'degraded_ratio': u'22.956', u'bytes_avail': 4955690496000}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 547, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:36,355.355 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:36,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1230 16 5415 5399 17.5544 0 - 3.63616 2014-07-01 16:36:37,098.098 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:36:37,098.098 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: finishing write tid 2 to mira0435601-439 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 1 to mira0435601-368 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: finishing write tid 1 to mira0435601-140 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 2 to mira0435601-368 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 4 to mira0435601-368 2014-07-01 16:36:37,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: finishing write tid 2 to mira0435601-140 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: finishing write tid 3 to mira0435601-140 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: finishing write tid 1 to mira0435601-291 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: finishing write tid 3 to mira0435601-291 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: finishing write tid 5 to mira0435601-140 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: finishing write tid 1 to mira0435601-93 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: finishing write tid 2 to mira0435601-93 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: finishing write tid 3 to mira0435601-93 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: done (15 left) 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: done (14 left) 2014-07-01 16:36:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: done (13 left) 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1566: done (12 left) 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1579: delete oid 59 current snap is 0 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: finishing write tid 5 to mira0435601-93 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: finishing write tid 3 to mira0435601-439 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: finishing write tid 5 to mira0435601-439 2014-07-01 16:36:37,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: finishing write tid 6 to mira0435601-439 2014-07-01 16:36:37,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 189 (ObjNum 1085 snap 0 seq_num 1085) dirty exists 2014-07-01 16:36:37,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: finishing write tid 6 to mira0435601-93 2014-07-01 16:36:37,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 91 (ObjNum 1084 snap 0 seq_num 1084) dirty exists 2014-07-01 16:36:37,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: expect (ObjNum 157 snap 0 seq_num 157) 2014-07-01 16:36:37,492.492 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1231 16 5415 5399 17.5402 0 - 3.63616 2014-07-01 16:36:38,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1232 16 5415 5399 17.5259 0 - 3.63616 2014-07-01 16:36:39,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1233 16 5415 5399 17.5117 0 - 3.63616 2014-07-01 16:36:40,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: done (12 left) 2014-07-01 16:36:40,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: done (11 left) 2014-07-01 16:36:40,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: done (10 left) 2014-07-01 16:36:40,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1579: done (9 left) 2014-07-01 16:36:40,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1580: read oid 151 snap -1 2014-07-01 16:36:40,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1581: read oid 385 snap -1 2014-07-01 16:36:40,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:36:40,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: seq_num 1087 ranges {3559424=487424,4046848=622592,4669440=180224} 2014-07-01 16:36:40,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: writing mira0435601-172 from 3559424 to 4046848 tid 1 2014-07-01 16:36:40,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: writing mira0435601-172 from 4046848 to 4669440 tid 2 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: writing mira0435601-172 from 4669440 to 4849664 tid 3 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 448 current snap is 0 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: seq_num 1088 ranges {4366336=761856,5128192=401408} 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: writing mira0435601-448 from 4366336 to 5128192 tid 1 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: writing mira0435601-448 from 5128192 to 5529600 tid 2 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1584: read oid 284 snap -1 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: seq_num 1089 ranges {3260416=786432,4046848=434176,4481024=688128,5169152=483328} 2014-07-01 16:36:40,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: writing mira0435601-288 from 3260416 to 4046848 tid 1 2014-07-01 16:36:40,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: writing mira0435601-288 from 4046848 to 4481024 tid 2 2014-07-01 16:36:40,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: writing mira0435601-288 from 4481024 to 5169152 tid 3 2014-07-01 16:36:40,427.427 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:40,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: writing mira0435601-288 from 5169152 to 5652480 tid 4 2014-07-01 16:36:40,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1586: read oid 98 snap -1 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 1 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 2 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1584: done (15 left) 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1587: delete oid 24 current snap is 0 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 3 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 4 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 6 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: finishing write tid 7 to mira0435601-288 2014-07-01 16:36:40,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 290 (ObjNum 1089 snap 0 seq_num 1089) dirty exists 2014-07-01 16:36:40,503.503 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1234 16 5415 5399 17.4975 0 - 3.63616 2014-07-01 16:36:41,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1235 16 5415 5399 17.4834 0 - 3.63616 2014-07-01 16:36:42,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1236 16 5415 5399 17.4692 0 - 3.63616 2014-07-01 16:36:42,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: finishing write tid 1 to mira0435601-45 2014-07-01 16:36:43,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 5 to mira0435601-368 2014-07-01 16:36:43,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 124 (ObjNum 1074 snap 0 seq_num 1074) dirty exists 2014-07-01 16:36:43,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: finishing write tid 6 to mira0435601-140 2014-07-01 16:36:43,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 107 (ObjNum 1076 snap 0 seq_num 1076) dirty exists 2014-07-01 16:36:43,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: finishing write tid 4 to mira0435601-291 2014-07-01 16:36:43,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 176 (ObjNum 1082 snap 0 seq_num 1082) dirty exists 2014-07-01 16:36:43,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 1 to mira0435601-448 2014-07-01 16:36:43,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 16:36:43,497.497 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1237 16 5433 5417 17.5133 8 0.137674 3.64505 2014-07-01 16:36:44,495.495 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1238 16 5440 5424 17.5217 28 11.0839 3.64616 2014-07-01 16:36:44,959.959 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:45,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: finishing write tid 2 to mira0435601-45 2014-07-01 16:36:45,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: expect (ObjNum 723 snap 0 seq_num 723) 2014-07-01 16:36:45,493.493 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1239 16 5442 5426 17.5141 8 1.58877 3.64528 2014-07-01 16:36:45,729.729 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:45,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: finishing write tid 3 to mira0435601-45 2014-07-01 16:36:45,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: finishing write tid 1 to mira0435601-449 2014-07-01 16:36:45,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: finishing write tid 5 to mira0435601-45 2014-07-01 16:36:45,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: finishing write tid 3 to mira0435601-449 2014-07-01 16:36:45,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: finishing write tid 1 to mira0435601-382 2014-07-01 16:36:45,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: finishing write tid 2 to mira0435601-382 2014-07-01 16:36:45,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: finishing write tid 3 to mira0435601-382 2014-07-01 16:36:45,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: finishing write tid 5 to mira0435601-382 2014-07-01 16:36:45,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1578: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 16:36:46,495.495 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:36:46.406877min lat: 0.057269 max lat: 294.525 avg lat: 3.64528 2014-07-01 16:36:46,495.495 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:36:46,495.495 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1240 16 5442 5426 17.4999 0 - 3.64528 2014-07-01 16:36:46,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: finishing write tid 6 to mira0435601-382 2014-07-01 16:36:46,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 88 (ObjNum 1083 snap 0 seq_num 1083) dirty exists 2014-07-01 16:36:47,273.273 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:47,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1241 16 5442 5426 17.4858 0 - 3.64528 2014-07-01 16:36:47,974.974 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:35:54.533688', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886811520, u'health': u'HEALTH_OK', u'kb_used': 9969532, u'store_stats': {u'bytes_total': 7657746, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463426}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2773/12221 objects degraded (22.690%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2773, u'num_pgs': 244, u'data_bytes': 23736389822, u'degraded_total': 12221, u'bytes_used': 43126628352, u'op_per_sec': 3, u'write_bytes_sec': 15085599, u'version': 1102, u'pgs_by_state': [{u'count': 35, u'state_name': u'active+recovery_wait'}, {u'count': 185, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.690', u'bytes_avail': 4955455909888}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 551, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:47,974.974 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:48,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1242 16 5442 5426 17.4718 0 - 3.64528 2014-07-01 16:36:49,008.008 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:36:49,008.008 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:49,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1243 16 5442 5426 17.4577 0 - 3.64528 2014-07-01 16:36:49,757.757 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:50,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 2 to mira0435601-448 2014-07-01 16:36:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 4 to mira0435601-448 2014-07-01 16:36:50,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:1580: expect (ObjNum 887 snap 0 seq_num 887) 2014-07-01 16:36:50,494.494 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1244 16 5444 5428 17.4501 1.6 0.145393 3.64498 2014-07-01 16:36:51,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1586: expect (ObjNum 1008 snap 0 seq_num 1008) 2014-07-01 16:36:51,497.497 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1245 16 5444 5428 17.4361 0 - 3.64498 2014-07-01 16:36:52,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1246 16 5444 5428 17.4221 0 - 3.64498 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: done (15 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: done (14 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: done (13 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: done (12 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: done (11 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: done (10 left) 2014-07-01 16:36:52,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1578: done (9 left) 2014-07-01 16:36:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1580: done (8 left) 2014-07-01 16:36:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: done (7 left) 2014-07-01 16:36:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1586: done (6 left) 2014-07-01 16:36:52,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:1587: done (5 left) 2014-07-01 16:36:52,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 231 current snap is 0 2014-07-01 16:36:52,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: seq_num 1091 ranges {1200128=581632} 2014-07-01 16:36:52,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: writing mira0435601-231 from 1200128 to 1781760 tid 1 2014-07-01 16:36:52,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: delete oid 115 current snap is 0 2014-07-01 16:36:53,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1247 16 5450 5434 17.4274 8 0.243506 3.64993 2014-07-01 16:36:54,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: finishing write tid 4 to mira0435601-449 2014-07-01 16:36:54,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 199 (ObjNum 1081 snap 0 seq_num 1081) dirty exists 2014-07-01 16:36:54,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: finishing write tid 6 to mira0435601-45 2014-07-01 16:36:54,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 177 (ObjNum 1080 snap 0 seq_num 1080) dirty exists 2014-07-01 16:36:54,294.294 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:54,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: finishing write tid 1 to mira0435601-231 2014-07-01 16:36:54,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: finishing write tid 3 to mira0435601-231 2014-07-01 16:36:54,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 5 to mira0435601-448 2014-07-01 16:36:54,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 166 (ObjNum 1088 snap 0 seq_num 1088) dirty exists 2014-07-01 16:36:54,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1248 16 5469 5453 17.4743 76 0.150615 3.66207 2014-07-01 16:36:55,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: finishing write tid 4 to mira0435601-231 2014-07-01 16:36:55,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 98 (ObjNum 1091 snap 0 seq_num 1091) dirty exists 2014-07-01 16:36:55,046.046 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:36:55,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1249 16 5473 5457 17.4731 16 0.689985 3.65967 2014-07-01 16:36:56,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1250 16 5477 5461 17.4719 16 1.47037 3.65804 2014-07-01 16:36:56,597.597 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:57,288.288 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2789/12261 objects degraded (22.747%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2789, u'num_pgs': 244, u'data_bytes': 23820275822, u'degraded_total': 12261, u'bytes_used': 43164401664, u'op_per_sec': 1, u'write_bytes_sec': 8221385, u'version': 1111, u'pgs_by_state': [{u'count': 35, u'state_name': u'active+recovery_wait'}, {u'count': 185, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.747', u'bytes_avail': 4955418136576}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 556, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:36:57,289.289 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:36:57,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1251 16 5477 5461 17.458 0 - 3.65804 2014-07-01 16:36:58,020.020 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:36:58,021.021 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:36:58,496.496 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1252 16 5477 5461 17.444 0 - 3.65804 2014-07-01 16:36:59,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1253 16 5478 5462 17.4333 1.33333 4.16595 3.65813 2014-07-01 16:37:00,316.316 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:00,520.520 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1254 16 5481 5465 17.4289 12 4.98541 3.65891 2014-07-01 16:37:01,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1255 16 5484 5468 17.4246 12 5.59391 3.66 2014-07-01 16:37:02,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1256 16 5492 5476 17.4362 32 7.87166 3.65989 2014-07-01 16:37:03,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1257 16 5492 5476 17.4223 0 - 3.65989 2014-07-01 16:37:04,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1258 16 5492 5476 17.4085 0 - 3.65989 2014-07-01 16:37:04,832.832 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:05,571.571 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1259 16 5492 5476 17.3947 0 - 3.65989 2014-07-01 16:37:05,573.573 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: done (6 left) 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: done (5 left) 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: done (4 left) 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: done (3 left) 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: done (2 left) 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1590: read oid 367 snap -1 2014-07-01 16:37:05,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1591: read oid 80 snap -1 2014-07-01 16:37:05,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: read oid 1 snap -1 2014-07-01 16:37:05,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1593: read oid 227 snap -1 2014-07-01 16:37:05,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 455 current snap is 0 2014-07-01 16:37:05,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: seq_num 1093 ranges {1191936=479232,1671168=417792,2088960=737280,2826240=49152} 2014-07-01 16:37:05,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: writing mira0435601-455 from 1191936 to 1671168 tid 1 2014-07-01 16:37:05,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: writing mira0435601-455 from 1671168 to 2088960 tid 2 2014-07-01 16:37:05,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: writing mira0435601-455 from 2088960 to 2826240 tid 3 2014-07-01 16:37:05,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: writing mira0435601-455 from 2826240 to 2875392 tid 4 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1595: read oid 400 snap -1 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 38 current snap is 0 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: seq_num 1094 ranges {0=749568,749568=598016,1347584=204800} 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: writing mira0435601-38 from 0 to 749568 tid 1 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: writing mira0435601-38 from 749568 to 1347584 tid 2 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: writing mira0435601-38 from 1347584 to 1552384 tid 3 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 358 current snap is 0 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: seq_num 1095 ranges {98304=376832} 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: writing mira0435601-358 from 98304 to 475136 tid 1 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1598: read oid 33 snap -1 2014-07-01 16:37:05,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1599: delete oid 322 current snap is 0 2014-07-01 16:37:06,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1591: expect (ObjNum 1052 snap 0 seq_num 1052) 2014-07-01 16:37:06,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:37:06.410845min lat: 0.057269 max lat: 294.525 avg lat: 3.6664 2014-07-01 16:37:06,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:37:06,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1260 16 5505 5489 17.4221 13 4.07905 3.6664 2014-07-01 16:37:07,436.436 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:07,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1261 16 5505 5489 17.4083 0 - 3.6664 2014-07-01 16:37:08,235.235 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:36:11.316608', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886799620, u'health': u'HEALTH_OK', u'kb_used': 9981432, u'store_stats': {u'bytes_total': 8706332, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463436}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2789/12346 objects degraded (22.590%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 9136628, u'degraded_objects': 2789, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 24005885962, u'degraded_total': 12346, u'bytes_used': 43725914112, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 33324691, u'op_per_sec': 8, u'version': 1121, u'pgs_by_state': [{u'count': 35, u'state_name': u'active+recovery_wait'}, {u'count': 185, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.590', u'bytes_avail': 4954856624128}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 561, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:08,235.235 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:08,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1262 16 5505 5489 17.3945 0 - 3.6664 2014-07-01 16:37:09,054.054 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:37:09,054.054 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:09,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1263 16 5505 5489 17.3807 0 - 3.6664 2014-07-01 16:37:09,880.880 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:10,498.498 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1264 16 5505 5489 17.367 0 - 3.6664 2014-07-01 16:37:11,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1265 16 5505 5489 17.3533 0 - 3.6664 2014-07-01 16:37:12,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1266 16 5505 5489 17.3396 0 - 3.6664 2014-07-01 16:37:13,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1267 16 5513 5497 17.3511 4.57143 11.3455 3.67572 2014-07-01 16:37:13,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: finishing write tid 1 to mira0435601-172 2014-07-01 16:37:13,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1593: expect (ObjNum 572 snap 0 seq_num 572) 2014-07-01 16:37:14,422.422 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:14,499.499 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1268 16 5537 5521 17.4131 96 0.412486 3.67408 2014-07-01 16:37:14,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1595: expect (ObjNum 938 snap 0 seq_num 938) 2014-07-01 16:37:15,275.275 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:15,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1269 16 5551 5535 17.4435 56 0.776474 3.66658 2014-07-01 16:37:16,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1270 16 5555 5539 17.4424 16 1.21596 3.66462 2014-07-01 16:37:17,024.024 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:17,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1271 16 5564 5548 17.457 36 0.410815 3.66039 2014-07-01 16:37:17,761.761 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2787/12366 objects degraded (22.538%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2787, u'num_pgs': 244, u'data_bytes': 24049512438, u'degraded_total': 12366, u'bytes_used': 43862679552, u'version': 1128, u'pgs_by_state': [{u'count': 35, u'state_name': u'active+recovery_wait'}, {u'count': 185, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.538', u'bytes_avail': 4954719858688}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 563, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:17,761.761 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: finishing write tid 2 to mira0435601-172 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: finishing write tid 3 to mira0435601-172 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: finishing write tid 1 to mira0435601-358 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: finishing write tid 3 to mira0435601-358 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: finishing write tid 5 to mira0435601-172 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 1 to mira0435601-455 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 2 to mira0435601-455 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 3 to mira0435601-455 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 4 to mira0435601-455 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 6 to mira0435601-455 2014-07-01 16:37:18,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1590: expect (ObjNum 869 snap 0 seq_num 869) 2014-07-01 16:37:18,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1272 16 5564 5548 17.4433 0 - 3.66039 2014-07-01 16:37:18,531.531 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:37:18,531.531 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:19,388.388 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:19,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1273 16 5579 5563 17.4767 30 0.074955 3.65597 2014-07-01 16:37:19,921.921 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_many_snaps ... ok 2014-07-01 16:37:20,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:1581: expect (ObjNum 384 snap 0 seq_num 384) 2014-07-01 16:37:20,500.500 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1274 16 5579 5563 17.463 0 - 3.65597 2014-07-01 16:37:21,501.501 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1275 16 5579 5563 17.4493 0 - 3.65597 2014-07-01 16:37:21,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: expect (ObjNum 610 snap 0 seq_num 610) 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: finishing write tid 1 to mira0435601-38 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: finishing write tid 2 to mira0435601-38 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: finishing write tid 3 to mira0435601-38 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: finishing write tid 5 to mira0435601-38 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: finishing write tid 6 to mira0435601-172 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 193 (ObjNum 1087 snap 0 seq_num 1087) dirty exists 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: finishing write tid 7 to mira0435601-455 2014-07-01 16:37:22,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 194 (ObjNum 1093 snap 0 seq_num 1093) dirty exists 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: finishing write tid 6 to mira0435601-38 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 205 (ObjNum 1094 snap 0 seq_num 1094) dirty exists 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: finishing write tid 4 to mira0435601-358 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 172 (ObjNum 1095 snap 0 seq_num 1095) dirty exists 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1581: done (11 left) 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: done (10 left) 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1590: done (9 left) 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1591: done (8 left) 2014-07-01 16:37:22,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: done (7 left) 2014-07-01 16:37:22,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1593: done (6 left) 2014-07-01 16:37:22,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: done (5 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1595: done (4 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: done (3 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: done (2 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1598: done (1 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1599: done (0 left) 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: read oid 430 snap -1 2014-07-01 16:37:22,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: delete oid 386 current snap is 0 2014-07-01 16:37:22,501.501 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1276 16 5579 5563 17.4356 0 - 3.65597 2014-07-01 16:37:23,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1277 16 5579 5563 17.4219 0 - 3.65597 2014-07-01 16:37:23,951.951 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:24,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1278 16 5579 5563 17.4083 0 - 3.65597 2014-07-01 16:37:24,667.667 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:25,501.501 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1279 16 5579 5563 17.3947 0 - 3.65597 2014-07-01 16:37:26,247.247 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:26,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:37:26.414539min lat: 0.057269 max lat: 294.525 avg lat: 3.65597 2014-07-01 16:37:26,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:37:26,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1280 16 5579 5563 17.3811 0 - 3.65597 2014-07-01 16:37:26,954.954 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'59 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2806/12518 objects degraded (22.416%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 16639655, u'degraded_objects': 2806, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 24369872683, u'degraded_total': 12518, u'bytes_used': 44895354880, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 33279311, u'op_per_sec': 10, u'version': 1136, u'pgs_by_state': [{u'count': 35, u'state_name': u'active+recovery_wait'}, {u'count': 185, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3090990, u'degraded_ratio': u'22.416', u'bytes_avail': 4953687183360}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 565, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:26,954.954 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:27,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: expect (ObjNum 690 snap 0 seq_num 690) 2014-07-01 16:37:27,373.373 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_protect_snap ... SKIP 2014-07-01 16:37:27,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1281 16 5594 5578 17.4144 7.5 8.23686 3.67117 2014-07-01 16:37:27,773.773 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:37:27,774.774 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:28,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1282 16 5601 5585 17.4226 28 0.060329 3.66777 2014-07-01 16:37:28,532.532 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:29,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1283 16 5606 5590 17.4246 20 0.101606 3.66464 2014-07-01 16:37:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: done (1 left) 2014-07-01 16:37:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: done (0 left) 2014-07-01 16:37:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: read oid 208 snap -1 2014-07-01 16:37:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1603: read oid 59 snap -1 2014-07-01 16:37:29,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:1604: read oid 454 snap -1 2014-07-01 16:37:29,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 494 current snap is 0 2014-07-01 16:37:29,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: seq_num 1098 ranges {0=598016,598016=421888,1019904=458752,1478656=466944,1945600=258048} 2014-07-01 16:37:29,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: writing mira0435601-494 from 0 to 598016 tid 1 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: writing mira0435601-494 from 598016 to 1019904 tid 2 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: writing mira0435601-494 from 1019904 to 1478656 tid 3 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: writing mira0435601-494 from 1478656 to 1945600 tid 4 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: writing mira0435601-494 from 1945600 to 2203648 tid 5 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 303 current snap is 0 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: seq_num 1099 ranges {425984=471040,897024=405504,1302528=684032,1986560=516096} 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: writing mira0435601-303 from 425984 to 897024 tid 1 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: writing mira0435601-303 from 897024 to 1302528 tid 2 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: writing mira0435601-303 from 1302528 to 1986560 tid 3 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: writing mira0435601-303 from 1986560 to 2502656 tid 4 2014-07-01 16:37:29,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 482 current snap is 0 2014-07-01 16:37:29,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: seq_num 1100 ranges {626688=389120} 2014-07-01 16:37:29,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: writing mira0435601-482 from 626688 to 1015808 tid 1 2014-07-01 16:37:29,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 16:37:29,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: seq_num 1101 ranges {3395584=417792,3813376=622592,4435968=581632} 2014-07-01 16:37:29,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: writing mira0435601-292 from 3395584 to 3813376 tid 1 2014-07-01 16:37:29,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: writing mira0435601-292 from 3813376 to 4435968 tid 2 2014-07-01 16:37:29,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: writing mira0435601-292 from 4435968 to 5017600 tid 3 2014-07-01 16:37:30,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 149 current snap is 0 2014-07-01 16:37:30,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: seq_num 1102 ranges {1867776=692224,2560000=802816} 2014-07-01 16:37:30,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: writing mira0435601-149 from 1867776 to 2560000 tid 1 2014-07-01 16:37:30,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: writing mira0435601-149 from 2560000 to 3362816 tid 2 2014-07-01 16:37:30,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1610: read oid 309 snap -1 2014-07-01 16:37:30,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: expect (ObjNum 904 snap 0 seq_num 904) 2014-07-01 16:37:30,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1284 16 5606 5590 17.4111 0 - 3.66464 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: finishing write tid 1 to mira0435601-292 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: finishing write tid 2 to mira0435601-292 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: finishing write tid 3 to mira0435601-292 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: finishing write tid 5 to mira0435601-292 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: finishing write tid 6 to mira0435601-292 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 230 (ObjNum 1101 snap 0 seq_num 1101) dirty exists 2014-07-01 16:37:31,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1610: expect (ObjNum 308 snap 0 seq_num 308) 2014-07-01 16:37:31,502.502 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1285 16 5606 5590 17.3975 0 - 3.66464 2014-07-01 16:37:32,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: done (8 left) 2014-07-01 16:37:32,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1603: done (7 left) 2014-07-01 16:37:32,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: done (6 left) 2014-07-01 16:37:32,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1610: done (5 left) 2014-07-01 16:37:32,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 194 current snap is 0 2014-07-01 16:37:32,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: seq_num 1103 ranges {5423104=561152,5984256=790528,6774784=12288} 2014-07-01 16:37:32,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: writing mira0435601-194 from 5423104 to 5984256 tid 1 2014-07-01 16:37:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: writing mira0435601-194 from 5984256 to 6774784 tid 2 2014-07-01 16:37:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: writing mira0435601-194 from 6774784 to 6787072 tid 3 2014-07-01 16:37:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 326 current snap is 0 2014-07-01 16:37:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: seq_num 1104 ranges {294912=188416} 2014-07-01 16:37:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: writing mira0435601-326 from 294912 to 483328 tid 1 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 249 current snap is 0 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: seq_num 1105 ranges {802816=512000,1314816=401408} 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: writing mira0435601-249 from 802816 to 1314816 tid 1 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: writing mira0435601-249 from 1314816 to 1716224 tid 2 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: read oid 217 snap -1 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1615: delete oid 218 current snap is 0 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: finishing write tid 1 to mira0435601-249 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: finishing write tid 2 to mira0435601-249 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: finishing write tid 4 to mira0435601-249 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: finishing write tid 5 to mira0435601-249 2014-07-01 16:37:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 158 (ObjNum 1105 snap 0 seq_num 1105) dirty exists 2014-07-01 16:37:32,503.503 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1286 16 5606 5590 17.384 0 - 3.66464 2014-07-01 16:37:33,047.047 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:33,503.503 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1287 16 5608 5592 17.3767 2 13.7729 3.66858 2014-07-01 16:37:33,816.816 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:33,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 1 to mira0435601-303 2014-07-01 16:37:34,171.171 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_read ... ok 2014-07-01 16:37:34,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1288 16 5623 5607 17.4098 60 0.588069 3.66707 2014-07-01 16:37:35,363.363 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:35,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1289 16 5624 5608 17.3994 4 0.109547 3.66643 2014-07-01 16:37:36,113.113 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:36:41.689868', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883283320, u'health': u'HEALTH_OK', u'kb_used': 13497732, u'store_stats': {u'bytes_total': 9755176, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3463704}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'34 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'57 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2774/12537 objects degraded (22.127%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2774, u'num_pgs': 244, u'data_bytes': 24417529643, u'degraded_total': 12537, u'bytes_used': 44992843776, u'op_per_sec': 3, u'version': 1143, u'pgs_by_state': [{u'count': 34, u'state_name': u'active+recovery_wait'}, {u'count': 187, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 718354, u'degraded_ratio': u'22.127', u'bytes_avail': 4953589694464}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 565, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:36,114.114 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:36,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 2 to mira0435601-303 2014-07-01 16:37:36,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 1 to mira0435601-494 2014-07-01 16:37:36,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 3 to mira0435601-303 2014-07-01 16:37:36,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1604: expect (ObjNum 453 snap 0 seq_num 453) 2014-07-01 16:37:36,503.503 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1290 16 5630 5614 17.4045 24 0.220339 3.6665 2014-07-01 16:37:36,867.867 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:37:36,867.867 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:37,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1291 16 5635 5619 17.4065 20 0.115821 3.66336 2014-07-01 16:37:37,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 2 to mira0435601-494 2014-07-01 16:37:37,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 4 to mira0435601-303 2014-07-01 16:37:37,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 3 to mira0435601-494 2014-07-01 16:37:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 4 to mira0435601-494 2014-07-01 16:37:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 6 to mira0435601-303 2014-07-01 16:37:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: finishing write tid 1 to mira0435601-149 2014-07-01 16:37:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: finishing write tid 2 to mira0435601-149 2014-07-01 16:37:37,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: finishing write tid 4 to mira0435601-149 2014-07-01 16:37:37,743.743 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:37,783.783 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_read_bad_offset ... ok 2014-07-01 16:37:37,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: finishing write tid 1 to mira0435601-194 2014-07-01 16:37:37,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: finishing write tid 2 to mira0435601-194 2014-07-01 16:37:37,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: finishing write tid 3 to mira0435601-194 2014-07-01 16:37:37,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: finishing write tid 5 to mira0435601-194 2014-07-01 16:37:37,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: finishing write tid 1 to mira0435601-326 2014-07-01 16:37:37,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: finishing write tid 3 to mira0435601-326 2014-07-01 16:37:37,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: finishing write tid 4 to mira0435601-326 2014-07-01 16:37:37,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 96 (ObjNum 1104 snap 0 seq_num 1104) dirty exists 2014-07-01 16:37:38,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1292 16 5643 5627 17.4178 32 0.093521 3.66386 2014-07-01 16:37:39,504.504 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1293 16 5643 5627 17.4043 0 - 3.66386 2014-07-01 16:37:40,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1294 16 5643 5627 17.3909 0 - 3.66386 2014-07-01 16:37:41,506.506 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1295 16 5643 5627 17.3774 0 - 3.66386 2014-07-01 16:37:42,270.270 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:42,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1296 16 5643 5627 17.364 0 - 3.66386 2014-07-01 16:37:42,999.999 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:43,523.523 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1297 16 5643 5627 17.3506 0 - 3.66386 2014-07-01 16:37:44,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1298 16 5643 5627 17.3373 0 - 3.66386 2014-07-01 16:37:44,541.541 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:45,266.266 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'57 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2785/12602 objects degraded (22.100%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 1393235, u'degraded_objects': 2785, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 24555269931, u'degraded_total': 12602, u'bytes_used': 45522235392, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 10233271, u'op_per_sec': 9, u'version': 1149, u'pgs_by_state': [{u'count': 33, u'state_name': u'active+recovery_wait'}, {u'count': 187, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 439127, u'degraded_ratio': u'22.100', u'bytes_avail': 4953060302848}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 565, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:45,267.267 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:45,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 5 to mira0435601-494 2014-07-01 16:37:45,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1299 16 5645 5629 17.3301 1.14286 11.691 3.66759 2014-07-01 16:37:45,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: finishing write tid 7 to mira0435601-303 2014-07-01 16:37:45,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 169 (ObjNum 1099 snap 0 seq_num 1099) dirty exists 2014-07-01 16:37:45,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: expect (ObjNum 985 snap 0 seq_num 985) 2014-07-01 16:37:46,123.123 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:37:46,123.123 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:46,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:37:46.418423min lat: 0.057269 max lat: 294.525 avg lat: 3.67519 2014-07-01 16:37:46,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:37:46,505.505 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1300 16 5675 5659 17.409 120 0.156442 3.67519 2014-07-01 16:37:46,909.909 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:47,509.509 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1301 16 5682 5666 17.4172 28 0.867601 3.67178 2014-07-01 16:37:48,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 7 to mira0435601-494 2014-07-01 16:37:48,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: finishing write tid 1 to mira0435601-482 2014-07-01 16:37:48,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: finishing write tid 3 to mira0435601-482 2014-07-01 16:37:48,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: finishing write tid 5 to mira0435601-149 2014-07-01 16:37:48,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 196 (ObjNum 1102 snap 0 seq_num 1102) dirty exists 2014-07-01 16:37:48,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: finishing write tid 6 to mira0435601-194 2014-07-01 16:37:48,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 227 (ObjNum 1103 snap 0 seq_num 1103) dirty exists 2014-07-01 16:37:48,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: finishing write tid 4 to mira0435601-482 2014-07-01 16:37:48,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 207 (ObjNum 1100 snap 0 seq_num 1100) dirty exists 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: finishing write tid 8 to mira0435601-494 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 205 (ObjNum 1098 snap 0 seq_num 1098) dirty exists 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1604: done (9 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: done (8 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: done (7 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: done (6 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: done (5 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: done (4 left) 2014-07-01 16:37:48,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: done (3 left) 2014-07-01 16:37:48,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: done (2 left) 2014-07-01 16:37:48,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: done (1 left) 2014-07-01 16:37:48,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1615: done (0 left) 2014-07-01 16:37:48,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 281 current snap is 0 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: seq_num 1107 ranges {2138112=409600,2547712=770048,3317760=577536,3895296=258048} 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: writing mira0435601-281 from 2138112 to 2547712 tid 1 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: writing mira0435601-281 from 2547712 to 3317760 tid 2 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: writing mira0435601-281 from 3317760 to 3895296 tid 3 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: writing mira0435601-281 from 3895296 to 4153344 tid 4 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1617: read oid 140 snap -1 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1618: read oid 70 snap -1 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1619: read oid 55 snap -1 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 45 current snap is 0 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: seq_num 1108 ranges {5984256=684032,6668288=442368,7110656=450560,7561216=311296} 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: writing mira0435601-45 from 5984256 to 6668288 tid 1 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: writing mira0435601-45 from 6668288 to 7110656 tid 2 2014-07-01 16:37:48,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: writing mira0435601-45 from 7110656 to 7561216 tid 3 2014-07-01 16:37:48,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: writing mira0435601-45 from 7561216 to 7872512 tid 4 2014-07-01 16:37:48,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1621: read oid 274 snap -1 2014-07-01 16:37:48,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: read oid 377 snap -1 2014-07-01 16:37:48,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1623: delete oid 445 current snap is 0 2014-07-01 16:37:48,506.506 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1302 16 5690 5674 17.4284 32 2.21274 3.66898 2014-07-01 16:37:49,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1303 15 5693 5678 17.4273 16 1.00078 3.66765 2014-07-01 16:37:50,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1304 16 5694 5678 17.4139 0 - 3.66765 2014-07-01 16:37:51,453.453 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:51,506.506 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1305 16 5694 5678 17.4006 0 - 3.66765 2014-07-01 16:37:52,223.223 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:52,506.506 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1306 16 5697 5681 17.3964 4 0.116276 3.6667 2014-07-01 16:37:53,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1618: expect (ObjNum 69 snap 0 seq_num 69) 2014-07-01 16:37:53,197.197 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_remove_snap ... ok 2014-07-01 16:37:53,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: expect (ObjNum 899 snap 0 seq_num 899) 2014-07-01 16:37:53,506.506 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1307 16 5707 5691 17.4137 40 0.149555 3.66875 2014-07-01 16:37:53,781.781 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:54,483.483 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:36:54.534026', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886777536, u'health': u'HEALTH_OK', u'kb_used': 10003516, u'store_stats': {u'bytes_total': 4500972, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'57 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'17 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2810/12711 objects degraded (22.107%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 1392645, u'degraded_objects': 2810, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 24784453542, u'degraded_total': 12711, u'bytes_used': 45576118272, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 23674972, u'op_per_sec': 5, u'version': 1157, u'pgs_by_state': [{u'count': 33, u'state_name': u'active+recovery_wait'}, {u'count': 187, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'degraded_ratio': u'22.107', u'bytes_avail': 4953006419968}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 567, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:37:54,484.484 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:37:54,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1308 16 5710 5694 17.4096 12 0.223754 3.66699 2014-07-01 16:37:55,260.260 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:37:55,260.260 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:37:55,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1309 16 5710 5694 17.3963 0 - 3.66699 2014-07-01 16:37:56,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1617: expect (ObjNum 1076 snap 0 seq_num 1076) 2014-07-01 16:37:56,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1310 16 5713 5697 17.3922 6 8.61778 3.66983 2014-07-01 16:37:56,769.769 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:37:57,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1311 16 5721 5705 17.4033 32 8.71154 3.66998 2014-07-01 16:37:58,507.507 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1312 16 5721 5705 17.39 0 - 3.66998 2014-07-01 16:37:59,509.509 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1313 16 5737 5721 17.4255 32 0.269146 3.66971 2014-07-01 16:37:59,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1617: done (7 left) 2014-07-01 16:37:59,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1618: done (6 left) 2014-07-01 16:37:59,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: done (5 left) 2014-07-01 16:37:59,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1623: done (4 left) 2014-07-01 16:37:59,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 8 current snap is 0 2014-07-01 16:37:59,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: seq_num 1110 ranges {229376=630784,860160=610304,1470464=192512} 2014-07-01 16:37:59,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: writing mira0435601-8 from 229376 to 860160 tid 1 2014-07-01 16:37:59,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: writing mira0435601-8 from 860160 to 1470464 tid 2 2014-07-01 16:37:59,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: writing mira0435601-8 from 1470464 to 1662976 tid 3 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1625: read oid 457 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: read oid 134 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1627: read oid 445 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: read oid 430 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1629: read oid 349 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1630: read oid 108 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1631: read oid 268 snap -1 2014-07-01 16:37:59,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:1632: delete oid 326 current snap is 0 2014-07-01 16:38:00,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: expect (ObjNum 133 snap 0 seq_num 133) 2014-07-01 16:38:00,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1314 16 5746 5730 17.4396 36 0.483523 3.66633 2014-07-01 16:38:01,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1629: expect (ObjNum 348 snap 0 seq_num 348) 2014-07-01 16:38:01,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 1 to mira0435601-45 2014-07-01 16:38:01,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 2 to mira0435601-45 2014-07-01 16:38:01,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1621: expect (ObjNum 799 snap 0 seq_num 799) 2014-07-01 16:38:01,409.409 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:01,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1315 16 5748 5732 17.4325 8 3.71102 3.6664 2014-07-01 16:38:02,157.157 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:02,516.516 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1316 16 5748 5732 17.4192 0 - 3.6664 2014-07-01 16:38:02,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 3 to mira0435601-45 2014-07-01 16:38:02,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:1625: expect (ObjNum 892 snap 0 seq_num 892) 2014-07-01 16:38:03,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1317 16 5748 5732 17.406 0 - 3.6664 2014-07-01 16:38:03,749.749 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:04,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1318 16 5748 5732 17.3928 0 - 3.6664 2014-07-01 16:38:05,508.508 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1319 16 5748 5732 17.3796 0 - 3.6664 2014-07-01 16:38:06,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:38:06.421817min lat: 0.057269 max lat: 294.525 avg lat: 3.6664 2014-07-01 16:38:06,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:38:06,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1320 16 5748 5732 17.3664 0 - 3.6664 2014-07-01 16:38:07,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1321 16 5752 5736 17.3654 2.66667 7.22253 3.669 2014-07-01 16:38:07,565.565 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:37:11.316972', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886765656, u'health': u'HEALTH_OK', u'kb_used': 10015396, u'store_stats': {u'bytes_total': 5549567, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'57 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'15 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2830/12813 objects degraded (22.087%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 2129367, u'degraded_objects': 2830, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 25000251302, u'degraded_total': 12813, u'bytes_used': 46092328960, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 15677831, u'op_per_sec': 5, u'version': 1163, u'pgs_by_state': [{u'count': 33, u'state_name': u'active+recovery_wait'}, {u'count': 187, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1509428, u'degraded_ratio': u'22.087', u'bytes_avail': 4952490209280}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 567, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:07,565.565 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:07,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:1621: done (12 left) 2014-07-01 16:38:07,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:1625: done (11 left) 2014-07-01 16:38:07,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: done (10 left) 2014-07-01 16:38:07,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1627: done (9 left) 2014-07-01 16:38:07,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1629: done (8 left) 2014-07-01 16:38:07,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1630: done (7 left) 2014-07-01 16:38:07,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1632: done (6 left) 2014-07-01 16:38:07,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1633: read oid 275 snap -1 2014-07-01 16:38:07,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1634: read oid 12 snap -1 2014-07-01 16:38:07,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 16:38:07,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: seq_num 1112 ranges {6619136=126976} 2014-07-01 16:38:07,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: writing mira0435601-250 from 6619136 to 6746112 tid 1 2014-07-01 16:38:07,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:1636: delete oid 270 current snap is 0 2014-07-01 16:38:07,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:1634: expect (ObjNum 11 snap 0 seq_num 11) 2014-07-01 16:38:08,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1631: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:38:08,255.255 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:08,255.255 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:08,509.509 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1322 16 5769 5753 17.4037 68 8.42339 3.67455 2014-07-01 16:38:09,229.229 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:09,509.509 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1323 16 5769 5753 17.3905 0 - 3.67455 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 1 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 2 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 3 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 4 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 6 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 4 to mira0435601-45 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 6 to mira0435601-45 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: finishing write tid 7 to mira0435601-281 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 113 (ObjNum 1107 snap 0 seq_num 1107) dirty exists 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: finishing write tid 7 to mira0435601-45 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 182 (ObjNum 1108 snap 0 seq_num 1108) dirty exists 2014-07-01 16:38:09,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1633: expect (ObjNum 274 snap 0 seq_num 274) 2014-07-01 16:38:10,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1324 16 5779 5763 17.4076 20 0.546801 3.67209 2014-07-01 16:38:10,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: expect (ObjNum 690 snap 0 seq_num 690) 2014-07-01 16:38:11,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1325 16 5779 5763 17.3945 0 - 3.67209 2014-07-01 16:38:12,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1326 16 5779 5763 17.3814 0 - 3.67209 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: finishing write tid 1 to mira0435601-8 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: finishing write tid 2 to mira0435601-8 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: finishing write tid 3 to mira0435601-8 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: finishing write tid 5 to mira0435601-8 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: done (9 left) 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: done (8 left) 2014-07-01 16:38:13,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: done (7 left) 2014-07-01 16:38:13,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1631: done (6 left) 2014-07-01 16:38:13,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1633: done (5 left) 2014-07-01 16:38:13,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1634: done (4 left) 2014-07-01 16:38:13,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1636: done (3 left) 2014-07-01 16:38:13,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 291 current snap is 0 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: seq_num 1114 ranges {2588672=724992} 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: writing mira0435601-291 from 2588672 to 3313664 tid 1 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1638: read oid 124 snap -1 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: seq_num 1115 ranges {0=749568,749568=577536,1327104=626688} 2014-07-01 16:38:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: writing mira0435601-327 from 0 to 749568 tid 1 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: writing mira0435601-327 from 749568 to 1327104 tid 2 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: writing mira0435601-327 from 1327104 to 1953792 tid 3 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 423 current snap is 0 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: seq_num 1116 ranges {4231168=622592,4853760=405504,5259264=745472,6004736=53248} 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: writing mira0435601-423 from 4231168 to 4853760 tid 1 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: writing mira0435601-423 from 4853760 to 5259264 tid 2 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: writing mira0435601-423 from 5259264 to 6004736 tid 3 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: writing mira0435601-423 from 6004736 to 6057984 tid 4 2014-07-01 16:38:13,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1641: delete oid 307 current snap is 0 2014-07-01 16:38:13,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1327 16 5779 5763 17.3683 0 - 3.67209 2014-07-01 16:38:13,775.775 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:14,383.383 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:14,510.510 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1328 16 5779 5763 17.3552 0 - 3.67209 2014-07-01 16:38:15,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1329 16 5779 5763 17.3421 0 - 3.67209 2014-07-01 16:38:16,098.098 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:16,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1330 16 5782 5766 17.3381 2 6.58997 3.67355 2014-07-01 16:38:16,838.838 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'57 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2828/12841 objects degraded (22.023%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2828, u'num_pgs': 244, u'data_bytes': 25072471974, u'degraded_total': 12841, u'bytes_used': 46488551424, u'op_per_sec': 5, u'write_bytes_sec': 16716901, u'version': 1168, u'pgs_by_state': [{u'count': 33, u'state_name': u'active+recovery_wait'}, {u'count': 187, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}], u'read_bytes_sec': 292831, u'degraded_ratio': u'22.023', u'bytes_avail': 4952093986816}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 568, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:16,839.839 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:17,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1331 16 5783 5767 17.3281 4 8.77996 3.67444 2014-07-01 16:38:17,604.604 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:17,605.605 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:18,379.379 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:18,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1332 16 5786 5770 17.3241 12 0.129199 3.67427 2014-07-01 16:38:18,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 1 to mira0435601-250 2014-07-01 16:38:18,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 3 to mira0435601-250 2014-07-01 16:38:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1638: expect (ObjNum 842 snap 0 seq_num 842) 2014-07-01 16:38:18,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: finishing write tid 1 to mira0435601-291 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: finishing write tid 3 to mira0435601-291 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 1 to mira0435601-423 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 2 to mira0435601-423 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 3 to mira0435601-423 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 4 to mira0435601-423 2014-07-01 16:38:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 6 to mira0435601-423 2014-07-01 16:38:19,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: finishing write tid 6 to mira0435601-8 2014-07-01 16:38:19,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 8 v 197 (ObjNum 1110 snap 0 seq_num 1110) dirty exists 2014-07-01 16:38:19,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1333 16 5806 5790 17.3711 80 0.44686 3.68043 2014-07-01 16:38:19,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 1 to mira0435601-327 2014-07-01 16:38:19,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 4 to mira0435601-250 2014-07-01 16:38:19,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 114 (ObjNum 1112 snap 0 seq_num 1112) dirty exists 2014-07-01 16:38:19,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 2 to mira0435601-327 2014-07-01 16:38:19,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 3 to mira0435601-327 2014-07-01 16:38:19,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 5 to mira0435601-327 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: done (7 left) 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: done (6 left) 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1638: done (5 left) 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1641: done (4 left) 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: read oid 203 snap -1 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: read oid 394 snap -1 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 241 current snap is 0 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: seq_num 1118 ranges {2244608=569344,2813952=573440,3387392=229376} 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: writing mira0435601-241 from 2244608 to 2813952 tid 1 2014-07-01 16:38:20,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: writing mira0435601-241 from 2813952 to 3387392 tid 2 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: writing mira0435601-241 from 3387392 to 3616768 tid 3 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 315 current snap is 0 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: seq_num 1119 ranges {0=196608} 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: writing mira0435601-315 from 0 to 196608 tid 1 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1646: read oid 50 snap -1 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: finishing write tid 4 to mira0435601-291 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 178 (ObjNum 1114 snap 0 seq_num 1114) dirty exists 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: finishing write tid 7 to mira0435601-423 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 177 (ObjNum 1116 snap 0 seq_num 1116) dirty exists 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: done (8 left) 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: done (7 left) 2014-07-01 16:38:20,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:1647: delete oid 283 current snap is 0 2014-07-01 16:38:20,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1334 16 5812 5796 17.376 24 2.20094 3.68004 2014-07-01 16:38:21,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1647: done (7 left) 2014-07-01 16:38:21,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 214 current snap is 0 2014-07-01 16:38:21,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: seq_num 1121 ranges {1449984=741376,2191360=614400,2805760=462848,3268608=528384} 2014-07-01 16:38:21,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: writing mira0435601-214 from 1449984 to 2191360 tid 1 2014-07-01 16:38:21,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: writing mira0435601-214 from 2191360 to 2805760 tid 2 2014-07-01 16:38:21,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: writing mira0435601-214 from 2805760 to 3268608 tid 3 2014-07-01 16:38:21,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: writing mira0435601-214 from 3268608 to 3796992 tid 4 2014-07-01 16:38:21,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1649: read oid 371 snap -1 2014-07-01 16:38:21,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1650: read oid 263 snap -1 2014-07-01 16:38:21,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: delete oid 162 current snap is 0 2014-07-01 16:38:21,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1649: expect (ObjNum 1023 snap 0 seq_num 1023) 2014-07-01 16:38:21,511.511 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1335 16 5816 5800 17.375 16 1.30294 3.67842 2014-07-01 16:38:22,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 6 to mira0435601-327 2014-07-01 16:38:22,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 167 (ObjNum 1115 snap 0 seq_num 1115) dirty exists 2014-07-01 16:38:22,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: expect (ObjNum 821 snap 0 seq_num 821) 2014-07-01 16:38:22,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1336 16 5828 5812 17.3979 48 0.229407 3.67431 2014-07-01 16:38:22,894.894 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:23,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1337 16 5833 5817 17.3999 20 2.93054 3.67397 2014-07-01 16:38:23,621.621 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:24,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1338 16 5833 5817 17.3869 0 - 3.67397 2014-07-01 16:38:25,172.172 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:25,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: finishing write tid 1 to mira0435601-241 2014-07-01 16:38:25,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: finishing write tid 2 to mira0435601-241 2014-07-01 16:38:25,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 1 to mira0435601-315 2014-07-01 16:38:25,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 3 to mira0435601-315 2014-07-01 16:38:25,512.512 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1339 16 5836 5820 17.3828 6 5.45211 3.67428 2014-07-01 16:38:25,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1646: expect (ObjNum 1017 snap 0 seq_num 1017) 2014-07-01 16:38:25,875.875 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'55 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2825/12975 objects degraded (21.773%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 15921049, u'degraded_objects': 2825, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 25357463484, u'degraded_total': 12975, u'bytes_used': 46552289280, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 59597905, u'op_per_sec': 22, u'version': 1175, u'pgs_by_state': [{u'count': 33, u'state_name': u'active+recovery_wait'}, {u'count': 189, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1308062, u'degraded_ratio': u'21.773', u'bytes_avail': 4952030248960}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 568, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:25,875.875 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:26,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:38:26.425572min lat: 0.057269 max lat: 294.525 avg lat: 3.67428 2014-07-01 16:38:26,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:38:26,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1340 16 5836 5820 17.3699 0 - 3.67428 2014-07-01 16:38:26,655.655 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:26,656.656 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:26,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: expect (ObjNum 982 snap 0 seq_num 982) 2014-07-01 16:38:27,517.517 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:27,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1341 16 5848 5832 17.3927 24 0.14534 3.67441 2014-07-01 16:38:28,060.060 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_remove_with_snap ... ok 2014-07-01 16:38:28,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: finishing write tid 3 to mira0435601-241 2014-07-01 16:38:28,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: finishing write tid 5 to mira0435601-241 2014-07-01 16:38:28,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:1650: expect (ObjNum 897 snap 0 seq_num 897) 2014-07-01 16:38:28,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1342 16 5851 5835 17.3887 12 0.090977 3.67263 2014-07-01 16:38:29,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 4 to mira0435601-315 2014-07-01 16:38:29,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 208 (ObjNum 1119 snap 0 seq_num 1119) dirty exists 2014-07-01 16:38:29,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: done (10 left) 2014-07-01 16:38:29,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: done (9 left) 2014-07-01 16:38:29,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: done (8 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: done (7 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1646: done (6 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1649: done (5 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1650: done (4 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: done (3 left) 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 269 current snap is 0 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: seq_num 1123 ranges {0=532480} 2014-07-01 16:38:29,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: writing mira0435601-269 from 0 to 532480 tid 1 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 477 current snap is 0 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: seq_num 1124 ranges {2015232=561152,2576384=610304,3186688=593920} 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: writing mira0435601-477 from 2015232 to 2576384 tid 1 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: writing mira0435601-477 from 2576384 to 3186688 tid 2 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: writing mira0435601-477 from 3186688 to 3780608 tid 3 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:38:29,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: seq_num 1125 ranges {3379200=589824} 2014-07-01 16:38:29,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1343 16 5857 5841 17.3936 24 2.74648 3.67501 2014-07-01 16:38:29,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: writing mira0435601-359 from 3379200 to 3969024 tid 1 2014-07-01 16:38:29,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:1655: read oid 184 snap -1 2014-07-01 16:38:29,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1619: expect (ObjNum 703 snap 0 seq_num 703) 2014-07-01 16:38:30,514.514 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1344 16 5862 5846 17.3955 20 0.68143 3.67228 2014-07-01 16:38:30,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: finishing write tid 6 to mira0435601-241 2014-07-01 16:38:30,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 181 (ObjNum 1118 snap 0 seq_num 1118) dirty exists 2014-07-01 16:38:30,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1619: done (6 left) 2014-07-01 16:38:30,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: done (5 left) 2014-07-01 16:38:30,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 419 current snap is 0 2014-07-01 16:38:30,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: seq_num 1126 ranges {2768896=704512,3473408=778240,4251648=24576} 2014-07-01 16:38:31,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: writing mira0435601-419 from 2768896 to 3473408 tid 1 2014-07-01 16:38:31,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: writing mira0435601-419 from 3473408 to 4251648 tid 2 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: writing mira0435601-419 from 4251648 to 4276224 tid 3 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: read oid 176 snap -1 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 33 current snap is 0 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: seq_num 1127 ranges {0=782336,782336=401408} 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: writing mira0435601-33 from 0 to 782336 tid 1 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: writing mira0435601-33 from 782336 to 1183744 tid 2 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: delete oid 451 current snap is 0 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: finishing write tid 1 to mira0435601-419 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: finishing write tid 2 to mira0435601-419 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: finishing write tid 3 to mira0435601-419 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: finishing write tid 5 to mira0435601-419 2014-07-01 16:38:31,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: finishing write tid 1 to mira0435601-33 2014-07-01 16:38:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: finishing write tid 2 to mira0435601-33 2014-07-01 16:38:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: finishing write tid 4 to mira0435601-33 2014-07-01 16:38:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: finishing write tid 5 to mira0435601-33 2014-07-01 16:38:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 197 (ObjNum 1127 snap 0 seq_num 1127) dirty exists 2014-07-01 16:38:31,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1345 16 5862 5846 17.3826 0 - 3.67228 2014-07-01 16:38:32,027.027 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:32,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: finishing write tid 6 to mira0435601-419 2014-07-01 16:38:32,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 234 (ObjNum 1126 snap 0 seq_num 1126) dirty exists 2014-07-01 16:38:32,513.513 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1346 16 5862 5846 17.3697 0 - 3.67228 2014-07-01 16:38:32,609.609 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:33,514.514 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1347 16 5862 5846 17.3568 0 - 3.67228 2014-07-01 16:38:34,334.334 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:34,514.514 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1348 16 5862 5846 17.3439 0 - 3.67228 2014-07-01 16:38:35,047.047 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:37:41.690258', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883228160, u'health': u'HEALTH_OK', u'kb_used': 13552892, u'store_stats': {u'bytes_total': 6598678, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'31 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'55 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2830/13018 objects degraded (21.739%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 4168426, u'degraded_objects': 2830, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 25448234795, u'degraded_total': 13018, u'bytes_used': 47005270016, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 7589874, u'op_per_sec': 6, u'version': 1182, u'pgs_by_state': [{u'count': 31, u'state_name': u'active+recovery_wait'}, {u'count': 188, u'state_name': u'active+clean'}, {u'count': 24, u'state_name': u'active+recovering'}, {u'count': 1, u'state_name': u'active+clean+scrubbing'}], u'read_bytes_sec': 1017, u'degraded_ratio': u'21.739', u'bytes_avail': 4951577268224}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:35,047.047 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:35,515.515 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1349 16 5862 5846 17.3311 0 - 3.67228 2014-07-01 16:38:35,784.784 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:35,784.784 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:36,514.514 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1350 16 5862 5846 17.3182 0 - 3.67228 2014-07-01 16:38:36,561.561 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:37,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1655: expect (ObjNum 183 snap 0 seq_num 183) 2014-07-01 16:38:37,516.516 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1351 16 5871 5855 17.3321 5.14286 7.00088 3.68173 2014-07-01 16:38:38,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: expect (ObjNum 717 snap 0 seq_num 717) 2014-07-01 16:38:38,515.515 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1352 16 5883 5867 17.3547 48 0.134205 3.67491 2014-07-01 16:38:39,515.515 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1353 16 5883 5867 17.3419 0 - 3.67491 2014-07-01 16:38:39,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1655: done (8 left) 2014-07-01 16:38:39,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: done (7 left) 2014-07-01 16:38:39,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: done (6 left) 2014-07-01 16:38:39,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: done (5 left) 2014-07-01 16:38:39,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: done (4 left) 2014-07-01 16:38:39,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1660: delete oid 324 current snap is 0 2014-07-01 16:38:39,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1660: done (4 left) 2014-07-01 16:38:39,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 431 current snap is 0 2014-07-01 16:38:39,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: seq_num 1130 ranges {0=552960,552960=757760,1310720=634880,1945600=212992} 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: writing mira0435601-431 from 0 to 552960 tid 1 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: writing mira0435601-431 from 552960 to 1310720 tid 2 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: writing mira0435601-431 from 1310720 to 1945600 tid 3 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: writing mira0435601-431 from 1945600 to 2158592 tid 4 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1662: read oid 399 snap -1 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 1 to mira0435601-431 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1663: read oid 303 snap -1 2014-07-01 16:38:39,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1664: read oid 70 snap -1 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: read oid 262 snap -1 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1666: delete oid 237 current snap is 0 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 2 to mira0435601-431 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 3 to mira0435601-431 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 4 to mira0435601-431 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 6 to mira0435601-431 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: finishing write tid 7 to mira0435601-431 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 202 (ObjNum 1130 snap 0 seq_num 1130) dirty exists 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: finishing write tid 1 to mira0435601-269 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: finishing write tid 3 to mira0435601-269 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 1 to mira0435601-214 2014-07-01 16:38:39,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 2 to mira0435601-214 2014-07-01 16:38:39,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 3 to mira0435601-214 2014-07-01 16:38:39,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 4 to mira0435601-214 2014-07-01 16:38:39,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 6 to mira0435601-214 2014-07-01 16:38:39,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: finishing write tid 1 to mira0435601-477 2014-07-01 16:38:39,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: finishing write tid 2 to mira0435601-477 2014-07-01 16:38:39,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: finishing write tid 3 to mira0435601-477 2014-07-01 16:38:39,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: finishing write tid 5 to mira0435601-477 2014-07-01 16:38:39,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: finishing write tid 1 to mira0435601-359 2014-07-01 16:38:39,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: finishing write tid 3 to mira0435601-359 2014-07-01 16:38:40,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: finishing write tid 4 to mira0435601-359 2014-07-01 16:38:40,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 172 (ObjNum 1125 snap 0 seq_num 1125) dirty exists 2014-07-01 16:38:40,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:1664: expect (ObjNum 69 snap 0 seq_num 69) 2014-07-01 16:38:40,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1663: expect (ObjNum 1099 snap 0 seq_num 1099) 2014-07-01 16:38:40,515.515 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1354 16 5902 5886 17.3852 38 1.98186 3.68024 2014-07-01 16:38:40,933.933 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:41,516.516 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1355 16 5907 5891 17.3871 20 1.254 3.67803 2014-07-01 16:38:41,655.655 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:41,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: expect (ObjNum 989 snap 0 seq_num 989) 2014-07-01 16:38:42,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1356 16 5907 5891 17.3743 0 - 3.67803 2014-07-01 16:38:42,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: finishing write tid 6 to mira0435601-477 2014-07-01 16:38:42,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 116 (ObjNum 1124 snap 0 seq_num 1124) dirty exists 2014-07-01 16:38:42,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: finishing write tid 7 to mira0435601-214 2014-07-01 16:38:42,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 110 (ObjNum 1121 snap 0 seq_num 1121) dirty exists 2014-07-01 16:38:42,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1662: expect (ObjNum 398 snap 0 seq_num 398) 2014-07-01 16:38:43,190.190 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:43,515.515 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1357 16 5908 5892 17.3645 2 2.0264 3.67775 2014-07-01 16:38:43,898.898 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'53 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2814/13100 objects degraded (21.481%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2814, u'num_pgs': 244, u'data_bytes': 25627197350, u'degraded_total': 13100, u'bytes_used': 47643873280, u'op_per_sec': 18, u'write_bytes_sec': 35132522, u'version': 1187, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 190, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}, {u'count': 1, u'state_name': u'active+clean+scrubbing'}], u'read_bytes_sec': 2662676, u'degraded_ratio': u'21.481', u'bytes_avail': 4950938664960}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:43,899.899 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: finishing write tid 4 to mira0435601-269 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 269 v 116 (ObjNum 1123 snap 0 seq_num 1123) dirty exists 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: done (9 left) 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: done (8 left) 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: done (7 left) 2014-07-01 16:38:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: done (6 left) 2014-07-01 16:38:44,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: done (5 left) 2014-07-01 16:38:44,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1662: done (4 left) 2014-07-01 16:38:44,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1663: done (3 left) 2014-07-01 16:38:44,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1664: done (2 left) 2014-07-01 16:38:44,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: done (1 left) 2014-07-01 16:38:44,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1666: done (0 left) 2014-07-01 16:38:44,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1667: read oid 251 snap -1 2014-07-01 16:38:44,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1668: delete oid 144 current snap is 0 2014-07-01 16:38:44,545.545 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_remove_with_watcher ... ok 2014-07-01 16:38:44,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1358 16 5915 5899 17.3723 28 0.680083 3.67611 2014-07-01 16:38:44,663.663 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:44,663.663 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:45,472.472 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:45,520.520 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1359 16 5915 5899 17.3595 0 - 3.67611 2014-07-01 16:38:46,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:38:46.429174min lat: 0.057269 max lat: 294.525 avg lat: 3.67611 2014-07-01 16:38:46,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:38:46,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1360 16 5915 5899 17.3467 0 - 3.67611 2014-07-01 16:38:47,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1361 16 5915 5899 17.334 0 - 3.67611 2014-07-01 16:38:48,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1362 16 5915 5899 17.3213 0 - 3.67611 2014-07-01 16:38:49,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1363 16 5915 5899 17.3086 0 - 3.67611 2014-07-01 16:38:49,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:1667: expect (ObjNum 250 snap 0 seq_num 250) 2014-07-01 16:38:49,845.845 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:50,189.189 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_resize ... ok 2014-07-01 16:38:50,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1364 16 5937 5921 17.3604 14.6667 0.433456 3.68226 2014-07-01 16:38:50,599.599 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:50,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1667: done (1 left) 2014-07-01 16:38:50,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1668: done (0 left) 2014-07-01 16:38:50,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 162 current snap is 0 2014-07-01 16:38:50,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: seq_num 1133 ranges {0=774144,774144=802816} 2014-07-01 16:38:51,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: writing mira0435601-162 from 0 to 774144 tid 1 2014-07-01 16:38:51,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: writing mira0435601-162 from 774144 to 1576960 tid 2 2014-07-01 16:38:51,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:38:51,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: seq_num 1134 ranges {0=401408} 2014-07-01 16:38:51,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: writing mira0435601-284 from 0 to 401408 tid 1 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 73 current snap is 0 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: seq_num 1135 ranges {1368064=270336} 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: writing mira0435601-73 from 1368064 to 1638400 tid 1 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: read oid 226 snap -1 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: delete oid 317 current snap is 0 2014-07-01 16:38:51,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: expect (ObjNum 1061 snap 0 seq_num 1061) 2014-07-01 16:38:51,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1365 16 5944 5928 17.3682 28 0.181431 3.67858 2014-07-01 16:38:52,136.136 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:52,517.517 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1366 16 5946 5930 17.3613 8 0.305043 3.67743 2014-07-01 16:38:52,842.842 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:37:54.534459', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886747280, u'health': u'HEALTH_OK', u'kb_used': 10033772, u'store_stats': {u'bytes_total': 8695276, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'53 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2816/13171 objects degraded (21.380%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 2455801, u'degraded_objects': 2816, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 25782615851, u'degraded_total': 13171, u'bytes_used': 48087486464, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 20465414, u'op_per_sec': 6, u'version': 1192, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 191, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 399, u'degraded_ratio': u'21.380', u'bytes_avail': 4950495051776}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:38:52,842.842 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:38:53,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1367 16 5946 5930 17.3486 0 - 3.67743 2014-07-01 16:38:53,606.606 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:38:53,606.606 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:54,374.374 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:54,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1368 16 5946 5930 17.3359 0 - 3.67743 2014-07-01 16:38:55,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1369 16 5946 5930 17.3233 0 - 3.67743 2014-07-01 16:38:56,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1370 16 5946 5930 17.3106 0 - 3.67743 2014-07-01 16:38:57,518.518 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1371 16 5962 5946 17.3447 12.8 0.162665 3.68198 2014-07-01 16:38:58,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1372 16 5962 5946 17.332 0 - 3.68198 2014-07-01 16:38:58,750.750 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:38:59,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: finishing write tid 1 to mira0435601-162 2014-07-01 16:38:59,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: finishing write tid 1 to mira0435601-284 2014-07-01 16:38:59,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: finishing write tid 2 to mira0435601-162 2014-07-01 16:38:59,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: finishing write tid 3 to mira0435601-284 2014-07-01 16:38:59,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: done (4 left) 2014-07-01 16:38:59,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: done (3 left) 2014-07-01 16:38:59,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 447 current snap is 0 2014-07-01 16:38:59,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: seq_num 1137 ranges {1253376=618496,1871872=786432,2658304=651264,3309568=167936} 2014-07-01 16:38:59,343.343 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: writing mira0435601-447 from 1253376 to 1871872 tid 1 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: writing mira0435601-447 from 1871872 to 2658304 tid 2 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: writing mira0435601-447 from 2658304 to 3309568 tid 3 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: writing mira0435601-447 from 3309568 to 3477504 tid 4 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1675: read oid 141 snap -1 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: finishing write tid 4 to mira0435601-284 2014-07-01 16:38:59,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 229 (ObjNum 1134 snap 0 seq_num 1134) dirty exists 2014-07-01 16:38:59,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: done (4 left) 2014-07-01 16:38:59,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 16:38:59,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: seq_num 1138 ranges {5808128=491520,6299648=720896,7020544=524288,7544832=360448} 2014-07-01 16:38:59,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: writing mira0435601-94 from 5808128 to 6299648 tid 1 2014-07-01 16:38:59,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: writing mira0435601-94 from 6299648 to 7020544 tid 2 2014-07-01 16:38:59,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: writing mira0435601-94 from 7020544 to 7544832 tid 3 2014-07-01 16:38:59,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: writing mira0435601-94 from 7544832 to 7905280 tid 4 2014-07-01 16:38:59,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1677: read oid 348 snap -1 2014-07-01 16:38:59,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 1 to mira0435601-447 2014-07-01 16:38:59,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 2 to mira0435601-447 2014-07-01 16:38:59,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 3 to mira0435601-447 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 4 to mira0435601-447 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 6 to mira0435601-447 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 440 current snap is 0 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: seq_num 1139 ranges {1101824=765952,1867776=548864,2416640=655360} 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: writing mira0435601-440 from 1101824 to 1867776 tid 1 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: writing mira0435601-440 from 1867776 to 2416640 tid 2 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: writing mira0435601-440 from 2416640 to 3072000 tid 3 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 262 current snap is 0 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: seq_num 1140 ranges {1998848=634880,2633728=782336,3416064=421888} 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: writing mira0435601-262 from 1998848 to 2633728 tid 1 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: writing mira0435601-262 from 2633728 to 3416064 tid 2 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: writing mira0435601-262 from 3416064 to 3837952 tid 3 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 71 current snap is 0 2014-07-01 16:38:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: seq_num 1141 ranges {1626112=712704,2338816=610304,2949120=192512} 2014-07-01 16:38:59,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1373 16 5974 5958 17.3544 24 7.73761 3.68589 2014-07-01 16:38:59,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: writing mira0435601-71 from 1626112 to 2338816 tid 1 2014-07-01 16:38:59,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: writing mira0435601-71 from 2338816 to 2949120 tid 2 2014-07-01 16:38:59,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: writing mira0435601-71 from 2949120 to 3141632 tid 3 2014-07-01 16:38:59,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: read oid 242 snap -1 2014-07-01 16:38:59,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:1677: expect (ObjNum 706 snap 0 seq_num 706) 2014-07-01 16:38:59,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1675: expect (ObjNum 1027 snap 0 seq_num 1027) 2014-07-01 16:39:00,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1374 16 5979 5963 17.3563 20 0.813743 3.68483 2014-07-01 16:39:01,070.070 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:01,519.519 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1375 16 5983 5967 17.3553 16 1.50073 3.68333 2014-07-01 16:39:01,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: expect (ObjNum 742 snap 0 seq_num 742) 2014-07-01 16:39:01,776.776 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014291', u'skew': u'-0.005221', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002740', u'skew': u'-0.017105', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:38:11.317376', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886743920, u'health': u'HEALTH_OK', u'kb_used': 10037132, u'store_stats': {u'bytes_total': 9743871, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'52 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'15 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2803/13217 objects degraded (21.208%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3869883, u'degraded_objects': 2803, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 25881181995, u'degraded_total': 13217, u'bytes_used': 48348205056, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 14705933, u'op_per_sec': 6, u'version': 1195, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 192, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1053636, u'degraded_ratio': u'21.208', u'bytes_avail': 4950234333184}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:01,776.776 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:02,497.497 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:39:02,498.498 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:02,519.519 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1376 16 5984 5968 17.3456 4 2.35969 3.68311 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: finishing write tid 4 to mira0435601-162 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 1 to mira0435601-73 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 3 to mira0435601-73 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 1 to mira0435601-94 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: finishing write tid 1 to mira0435601-71 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: finishing write tid 2 to mira0435601-71 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: finishing write tid 3 to mira0435601-71 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: finishing write tid 5 to mira0435601-71 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: finishing write tid 6 to mira0435601-71 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 162 (ObjNum 1141 snap 0 seq_num 1141) dirty exists 2014-07-01 16:39:03,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: finishing write tid 1 to mira0435601-440 2014-07-01 16:39:03,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 2 to mira0435601-94 2014-07-01 16:39:03,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1675: done (9 left) 2014-07-01 16:39:03,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1677: done (8 left) 2014-07-01 16:39:03,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: done (7 left) 2014-07-01 16:39:03,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: done (6 left) 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1682: delete oid 150 current snap is 0 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: finishing write tid 2 to mira0435601-440 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 3 to mira0435601-94 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 4 to mira0435601-94 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: finishing write tid 3 to mira0435601-440 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: finishing write tid 5 to mira0435601-440 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 6 to mira0435601-94 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: finishing write tid 1 to mira0435601-262 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: finishing write tid 2 to mira0435601-262 2014-07-01 16:39:03,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: finishing write tid 3 to mira0435601-262 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: finishing write tid 5 to mira0435601-262 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: finishing write tid 7 to mira0435601-447 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 226 (ObjNum 1137 snap 0 seq_num 1137) dirty exists 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: finishing write tid 5 to mira0435601-162 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 175 (ObjNum 1133 snap 0 seq_num 1133) dirty exists 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 4 to mira0435601-73 2014-07-01 16:39:03,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 177 (ObjNum 1135 snap 0 seq_num 1135) dirty exists 2014-07-01 16:39:03,269.269 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:03,519.519 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1377 16 5984 5968 17.333 0 - 3.68311 2014-07-01 16:39:04,520.520 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1378 16 5984 5968 17.3204 0 - 3.68311 2014-07-01 16:39:05,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1379 16 5991 5975 17.3281 9.33333 5.48216 3.68544 2014-07-01 16:39:06,618.618 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:39:06.433034min lat: 0.057269 max lat: 294.525 avg lat: 3.68656 2014-07-01 16:39:06,618.618 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:39:06,618.618 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1380 16 5994 5978 17.3243 12 5.41857 3.68656 2014-07-01 16:39:07,522.522 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1381 16 5994 5978 17.3117 0 - 3.68656 2014-07-01 16:39:08,055.055 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:08,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1382 16 5994 5978 17.2992 0 - 3.68656 2014-07-01 16:39:08,660.660 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:08,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: finishing write tid 7 to mira0435601-94 2014-07-01 16:39:08,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 187 (ObjNum 1138 snap 0 seq_num 1138) dirty exists 2014-07-01 16:39:09,522.522 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1383 16 6012 5996 17.3388 24 0.767142 3.68858 2014-07-01 16:39:10,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1384 16 6015 5999 17.3349 12 0.12444 3.68699 2014-07-01 16:39:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: finishing write tid 6 to mira0435601-262 2014-07-01 16:39:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 233 (ObjNum 1140 snap 0 seq_num 1140) dirty exists 2014-07-01 16:39:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: finishing write tid 6 to mira0435601-440 2014-07-01 16:39:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 111 (ObjNum 1139 snap 0 seq_num 1139) dirty exists 2014-07-01 16:39:10,735.735 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:11,529.529 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2813/13322 objects degraded (21.115%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2813, u'num_pgs': 244, u'data_bytes': 26107291799, u'degraded_total': 13322, u'bytes_used': 48836157440, u'op_per_sec': 0, u'version': 1202, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 503, u'degraded_ratio': u'21.115', u'bytes_avail': 4949746380800}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:11,529.529 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:11,529.529 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1385 16 6018 6002 17.331 12 2.64387 3.68646 2014-07-01 16:39:12,301.301 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:39:12,302.302 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:12,522.522 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1386 16 6018 6002 17.3185 0 - 3.68646 2014-07-01 16:39:13,124.124 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:13,521.521 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1387 16 6021 6005 17.3147 6 7.17864 3.6872 2014-07-01 16:39:14,522.522 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1388 16 6021 6005 17.3022 0 - 3.6872 2014-07-01 16:39:15,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1389 16 6023 6007 17.2955 4 5.67585 3.6879 2014-07-01 16:39:16,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: done (6 left) 2014-07-01 16:39:16,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: done (5 left) 2014-07-01 16:39:16,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: done (4 left) 2014-07-01 16:39:16,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: done (3 left) 2014-07-01 16:39:16,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: done (2 left) 2014-07-01 16:39:16,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: done (1 left) 2014-07-01 16:39:16,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1682: done (0 left) 2014-07-01 16:39:16,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 240 current snap is 0 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: seq_num 1143 ranges {1011712=454656,1466368=778240,2244608=24576} 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: writing mira0435601-240 from 1011712 to 1466368 tid 1 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: writing mira0435601-240 from 1466368 to 2244608 tid 2 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: writing mira0435601-240 from 2244608 to 2269184 tid 3 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 91 current snap is 0 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: seq_num 1144 ranges {2351104=81920} 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: writing mira0435601-91 from 2351104 to 2433024 tid 1 2014-07-01 16:39:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 457 current snap is 0 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: seq_num 1145 ranges {2109440=512000,2621440=589824,3211264=212992} 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: writing mira0435601-457 from 2109440 to 2621440 tid 1 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: writing mira0435601-457 from 2621440 to 3211264 tid 2 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: writing mira0435601-457 from 3211264 to 3424256 tid 3 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 425 current snap is 0 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: seq_num 1146 ranges {2850816=675840} 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: writing mira0435601-425 from 2850816 to 3526656 tid 1 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1687: read oid 193 snap -1 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1688: read oid 285 snap -1 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: seq_num 1147 ranges {4583424=704512,5287936=757760,6045696=446464} 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: writing mira0435601-58 from 4583424 to 5287936 tid 1 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: writing mira0435601-58 from 5287936 to 6045696 tid 2 2014-07-01 16:39:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: writing mira0435601-58 from 6045696 to 6492160 tid 3 2014-07-01 16:39:16,724.724 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1390 16 6027 6011 17.2946 16 7.13478 3.68799 2014-07-01 16:39:16,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 165 current snap is 0 2014-07-01 16:39:16,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: seq_num 1148 ranges {3088384=733184} 2014-07-01 16:39:16,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: writing mira0435601-165 from 3088384 to 3821568 tid 1 2014-07-01 16:39:16,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 224 current snap is 0 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: seq_num 1149 ranges {1134592=446464,1581056=569344,2150400=438272,2588672=196608} 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: writing mira0435601-224 from 1134592 to 1581056 tid 1 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: writing mira0435601-224 from 1581056 to 2150400 tid 2 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: writing mira0435601-224 from 2150400 to 2588672 tid 3 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: writing mira0435601-224 from 2588672 to 2785280 tid 4 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1692: read oid 394 snap -1 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 244 current snap is 0 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: seq_num 1150 ranges {0=524288,524288=507904,1032192=294912} 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: writing mira0435601-244 from 0 to 524288 tid 1 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: writing mira0435601-244 from 524288 to 1032192 tid 2 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: writing mira0435601-244 from 1032192 to 1327104 tid 3 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: seq_num 1151 ranges {1556480=151552} 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: writing mira0435601-48 from 1556480 to 1708032 tid 1 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 64 current snap is 0 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: seq_num 1152 ranges {1560576=667648,2228224=401408} 2014-07-01 16:39:16,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: writing mira0435601-64 from 1560576 to 2228224 tid 1 2014-07-01 16:39:16,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: writing mira0435601-64 from 2228224 to 2629632 tid 2 2014-07-01 16:39:16,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 191 current snap is 0 2014-07-01 16:39:16,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: seq_num 1153 ranges {2592768=696320,3289088=798720,4087808=483328,4571136=323584} 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0435601-191 from 2592768 to 3289088 tid 1 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0435601-191 from 3289088 to 4087808 tid 2 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0435601-191 from 4087808 to 4571136 tid 3 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0435601-191 from 4571136 to 4894720 tid 4 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 325 current snap is 0 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: seq_num 1154 ranges {466944=724992,1191936=401408} 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: writing mira0435601-325 from 466944 to 1191936 tid 1 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: writing mira0435601-325 from 1191936 to 1593344 tid 2 2014-07-01 16:39:17,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: read oid 185 snap -1 2014-07-01 16:39:17,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:17,523.523 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1391 16 6028 6012 17.285 4 10.9193 3.6892 2014-07-01 16:39:17,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: finishing write tid 1 to mira0435601-48 2014-07-01 16:39:17,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: finishing write tid 3 to mira0435601-48 2014-07-01 16:39:17,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 1 to mira0435601-165 2014-07-01 16:39:17,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 3 to mira0435601-165 2014-07-01 16:39:17,908.908 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:17,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:1688: expect (ObjNum 284 snap 0 seq_num 284) 2014-07-01 16:39:18,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1692: expect (ObjNum 821 snap 0 seq_num 821) 2014-07-01 16:39:18,493.493 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:18,522.522 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1392 16 6036 6020 17.2956 32 0.190818 3.69281 2014-07-01 16:39:19,562.562 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1393 16 6036 6020 17.2832 0 - 3.69281 2014-07-01 16:39:20,244.244 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:20,523.523 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1394 16 6044 6028 17.2937 16 4.17768 3.69538 2014-07-01 16:39:20,944.944 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'5 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2817/13385 objects degraded (21.046%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 1668852, u'degraded_objects': 2817, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 26250999713, u'degraded_total': 13385, u'bytes_used': 49033560064, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 10258648, u'op_per_sec': 4, u'version': 1207, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 328, u'degraded_ratio': u'21.046', u'bytes_avail': 4949548978176}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:20,945.945 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:21,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: expect (ObjNum 184 snap 0 seq_num 184) 2014-07-01 16:39:21,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 4 to mira0435601-165 2014-07-01 16:39:21,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 196 (ObjNum 1148 snap 0 seq_num 1148) dirty exists 2014-07-01 16:39:21,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: finishing write tid 4 to mira0435601-48 2014-07-01 16:39:21,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 292 (ObjNum 1151 snap 0 seq_num 1151) dirty exists 2014-07-01 16:39:21,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1687: expect (ObjNum 1060 snap 0 seq_num 1060) 2014-07-01 16:39:21,523.523 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1395 16 6050 6034 17.2985 24 0.519909 3.69241 2014-07-01 16:39:21,685.685 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:39:21,686.686 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:22,493.493 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:22,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1396 16 6050 6034 17.2862 0 - 3.69241 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: finishing write tid 1 to mira0435601-457 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: finishing write tid 2 to mira0435601-457 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: finishing write tid 3 to mira0435601-457 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: finishing write tid 5 to mira0435601-457 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: finishing write tid 1 to mira0435601-91 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: finishing write tid 3 to mira0435601-91 2014-07-01 16:39:22,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1687: done (15 left) 2014-07-01 16:39:22,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1688: done (14 left) 2014-07-01 16:39:22,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: done (13 left) 2014-07-01 16:39:22,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1692: done (12 left) 2014-07-01 16:39:22,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: done (11 left) 2014-07-01 16:39:22,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: done (10 left) 2014-07-01 16:39:22,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 57 current snap is 0 2014-07-01 16:39:23,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: seq_num 1155 ranges {1028096=487424} 2014-07-01 16:39:23,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: writing mira0435601-57 from 1028096 to 1515520 tid 1 2014-07-01 16:39:23,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 441 current snap is 0 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: seq_num 1156 ranges {0=516096,516096=548864,1064960=655360} 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: writing mira0435601-441 from 0 to 516096 tid 1 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: writing mira0435601-441 from 516096 to 1064960 tid 2 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: writing mira0435601-441 from 1064960 to 1720320 tid 3 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1701: read oid 38 snap -1 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 153 current snap is 0 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: seq_num 1157 ranges {106496=540672} 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: writing mira0435601-153 from 106496 to 647168 tid 1 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 332 current snap is 0 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: seq_num 1158 ranges {536576=413696} 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: writing mira0435601-332 from 536576 to 950272 tid 1 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1704: read oid 19 snap -1 2014-07-01 16:39:23,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:23,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1397 16 6050 6034 17.2738 0 - 3.69241 2014-07-01 16:39:24,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: finishing write tid 1 to mira0435601-441 2014-07-01 16:39:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: finishing write tid 2 to mira0435601-441 2014-07-01 16:39:24,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: finishing write tid 3 to mira0435601-441 2014-07-01 16:39:24,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: finishing write tid 5 to mira0435601-441 2014-07-01 16:39:24,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: finishing write tid 6 to mira0435601-441 2014-07-01 16:39:24,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 200 (ObjNum 1156 snap 0 seq_num 1156) dirty exists 2014-07-01 16:39:24,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: done (15 left) 2014-07-01 16:39:24,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 262 current snap is 0 2014-07-01 16:39:24,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: seq_num 1159 ranges {3837952=737280,4575232=401408} 2014-07-01 16:39:24,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: writing mira0435601-262 from 3837952 to 4575232 tid 1 2014-07-01 16:39:24,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: writing mira0435601-262 from 4575232 to 4976640 tid 2 2014-07-01 16:39:24,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:24,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1398 16 6053 6037 17.27 4 0.106899 3.6911 2014-07-01 16:39:25,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1399 16 6053 6037 17.2577 0 - 3.6911 2014-07-01 16:39:26,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:39:26.437035min lat: 0.057269 max lat: 294.525 avg lat: 3.6911 2014-07-01 16:39:26,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:39:26,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1400 16 6053 6037 17.2453 0 - 3.6911 2014-07-01 16:39:27,018.018 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:27,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: finishing write tid 1 to mira0435601-425 2014-07-01 16:39:27,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: finishing write tid 3 to mira0435601-425 2014-07-01 16:39:27,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: finishing write tid 1 to mira0435601-58 2014-07-01 16:39:27,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: finishing write tid 2 to mira0435601-58 2014-07-01 16:39:27,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 1 to mira0435601-224 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: finishing write tid 3 to mira0435601-58 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 2 to mira0435601-224 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 1 to mira0435601-244 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 2 to mira0435601-244 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: finishing write tid 1 to mira0435601-325 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: finishing write tid 5 to mira0435601-58 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 3 to mira0435601-224 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 4 to mira0435601-224 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: finishing write tid 2 to mira0435601-325 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 3 to mira0435601-244 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 6 to mira0435601-224 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: finishing write tid 1 to mira0435601-64 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 5 to mira0435601-244 2014-07-01 16:39:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: finishing write tid 2 to mira0435601-64 2014-07-01 16:39:27,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: finishing write tid 1 to mira0435601-240 2014-07-01 16:39:27,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: finishing write tid 2 to mira0435601-240 2014-07-01 16:39:27,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: finishing write tid 4 to mira0435601-325 2014-07-01 16:39:27,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: finishing write tid 3 to mira0435601-240 2014-07-01 16:39:27,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: finishing write tid 5 to mira0435601-240 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: finishing write tid 4 to mira0435601-64 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 1 to mira0435601-191 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 2 to mira0435601-191 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 3 to mira0435601-191 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 4 to mira0435601-191 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 6 to mira0435601-191 2014-07-01 16:39:27,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1701: expect (ObjNum 1094 snap 0 seq_num 1094) 2014-07-01 16:39:27,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1401 16 6066 6050 17.2701 17.3333 0.337678 3.69864 2014-07-01 16:39:27,735.735 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 1 to mira0435601-262 2014-07-01 16:39:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1701: done (15 left) 2014-07-01 16:39:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: read oid 443 snap -1 2014-07-01 16:39:28,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:28,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1402 16 6072 6056 17.2749 24 1.3085 3.6957 2014-07-01 16:39:29,272.272 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:29,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: finishing write tid 1 to mira0435601-57 2014-07-01 16:39:29,524.524 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1403 16 6076 6060 17.274 16 9.59412 3.69671 2014-07-01 16:39:29,980.980 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'17 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2827/13436 objects degraded (21.040%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2827, u'num_pgs': 244, u'data_bytes': 26362894241, u'degraded_total': 13436, u'bytes_used': 49320763392, u'op_per_sec': 5, u'write_bytes_sec': 15477823, u'version': 1212, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'degraded_ratio': u'21.040', u'bytes_avail': 4949261774848}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:29,980.980 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:30,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: finishing write tid 3 to mira0435601-57 2014-07-01 16:39:30,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 2 to mira0435601-262 2014-07-01 16:39:30,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 4 to mira0435601-262 2014-07-01 16:39:30,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1704: done (15 left) 2014-07-01 16:39:30,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1707: read oid 62 snap -1 2014-07-01 16:39:30,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:30,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1707: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:39:30,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1404 16 6083 6067 17.2817 28 5.98467 3.69816 2014-07-01 16:39:30,707.707 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:39:30,707.707 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:31,532.532 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:31,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: expect (ObjNum 1020 snap 0 seq_num 1020) 2014-07-01 16:39:31,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1405 16 6092 6076 17.295 36 0.551326 3.69523 2014-07-01 16:39:32,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1406 16 6095 6079 17.2912 12 0.169071 3.69356 2014-07-01 16:39:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: done (15 left) 2014-07-01 16:39:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1707: done (14 left) 2014-07-01 16:39:32,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1708: read oid 293 snap -1 2014-07-01 16:39:32,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 471 current snap is 0 2014-07-01 16:39:32,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: seq_num 1160 ranges {2117632=761856,2879488=528384,3407872=708608,4116480=368640} 2014-07-01 16:39:32,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: writing mira0435601-471 from 2117632 to 2879488 tid 1 2014-07-01 16:39:32,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: writing mira0435601-471 from 2879488 to 3407872 tid 2 2014-07-01 16:39:32,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: writing mira0435601-471 from 3407872 to 4116480 tid 3 2014-07-01 16:39:32,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: writing mira0435601-471 from 4116480 to 4485120 tid 4 2014-07-01 16:39:32,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:33,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 1 to mira0435601-471 2014-07-01 16:39:33,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 2 to mira0435601-471 2014-07-01 16:39:33,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 3 to mira0435601-471 2014-07-01 16:39:33,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 4 to mira0435601-471 2014-07-01 16:39:33,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 6 to mira0435601-471 2014-07-01 16:39:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: finishing write tid 7 to mira0435601-471 2014-07-01 16:39:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 293 (ObjNum 1160 snap 0 seq_num 1160) dirty exists 2014-07-01 16:39:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: done (15 left) 2014-07-01 16:39:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1710: read oid 475 snap -1 2014-07-01 16:39:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:33,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1407 16 6095 6079 17.2789 0 - 3.69356 2014-07-01 16:39:34,525.525 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1408 16 6095 6079 17.2666 0 - 3.69356 2014-07-01 16:39:35,526.526 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1409 16 6095 6079 17.2544 0 - 3.69356 2014-07-01 16:39:35,900.900 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:36,526.526 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1410 16 6095 6079 17.2422 0 - 3.69356 2014-07-01 16:39:36,651.651 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:37,061.061 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_resize_bytes ... ok 2014-07-01 16:39:37,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: finishing write tid 6 to mira0435601-457 2014-07-01 16:39:37,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 191 (ObjNum 1145 snap 0 seq_num 1145) dirty exists 2014-07-01 16:39:37,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: finishing write tid 4 to mira0435601-91 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 91 v 199 (ObjNum 1144 snap 0 seq_num 1144) dirty exists 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: done (15 left) 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: done (14 left) 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1711: read oid 472 snap -1 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 179 current snap is 0 2014-07-01 16:39:37,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: seq_num 1161 ranges {2076672=634880,2711552=757760,3469312=532480,4001792=315392} 2014-07-01 16:39:37,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: writing mira0435601-179 from 2076672 to 2711552 tid 1 2014-07-01 16:39:37,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: writing mira0435601-179 from 2711552 to 3469312 tid 2 2014-07-01 16:39:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: writing mira0435601-179 from 3469312 to 4001792 tid 3 2014-07-01 16:39:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: writing mira0435601-179 from 4001792 to 4317184 tid 4 2014-07-01 16:39:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: finishing write tid 1 to mira0435601-332 2014-07-01 16:39:37,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: finishing write tid 3 to mira0435601-332 2014-07-01 16:39:37,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: finishing write tid 1 to mira0435601-153 2014-07-01 16:39:37,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1411 16 6105 6089 17.2583 8 0.386154 3.70036 2014-07-01 16:39:37,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1711: expect (ObjNum 471 snap 0 seq_num 471) 2014-07-01 16:39:38,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1711: done (15 left) 2014-07-01 16:39:38,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1713: read oid 116 snap -1 2014-07-01 16:39:38,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:38,187.187 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:38,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1713: expect (ObjNum 900 snap 0 seq_num 900) 2014-07-01 16:39:38,527.527 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1412 16 6111 6095 17.263 24 0.738604 3.69936 2014-07-01 16:39:38,891.891 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:38:41.690676', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883199724, u'health': u'HEALTH_OK', u'kb_used': 13581328, u'store_stats': {u'bytes_total': 8695830, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'82 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2828/13525 objects degraded (20.909%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12535377, u'degraded_objects': 2828, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 26549643163, u'degraded_total': 13525, u'bytes_used': 49851416576, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 21995738, u'op_per_sec': 11, u'version': 1218, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 909617, u'degraded_ratio': u'20.909', u'bytes_avail': 4948731121664}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:38,892.892 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:39,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1713: done (15 left) 2014-07-01 16:39:39,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1714: read oid 447 snap -1 2014-07-01 16:39:39,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:39:39,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1714: expect (ObjNum 1137 snap 0 seq_num 1137) 2014-07-01 16:39:39,526.526 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1413 16 6111 6095 17.2508 0 - 3.69936 2014-07-01 16:39:40,321.321 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:39:40,321.321 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:40,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1414 16 6124 6108 17.2754 26 2.75687 3.70206 2014-07-01 16:39:41,066.066 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:41,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: finishing write tid 6 to mira0435601-240 2014-07-01 16:39:41,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 119 (ObjNum 1143 snap 0 seq_num 1143) dirty exists 2014-07-01 16:39:41,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 6 to mira0435601-244 2014-07-01 16:39:41,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 160 (ObjNum 1150 snap 0 seq_num 1150) dirty exists 2014-07-01 16:39:41,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: finishing write tid 6 to mira0435601-58 2014-07-01 16:39:41,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 115 (ObjNum 1147 snap 0 seq_num 1147) dirty exists 2014-07-01 16:39:41,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 7 to mira0435601-191 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 192 (ObjNum 1153 snap 0 seq_num 1153) dirty exists 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: finishing write tid 7 to mira0435601-224 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 204 (ObjNum 1149 snap 0 seq_num 1149) dirty exists 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: finishing write tid 5 to mira0435601-64 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 172 (ObjNum 1152 snap 0 seq_num 1152) dirty exists 2014-07-01 16:39:41,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1710: expect (ObjNum 785 snap 0 seq_num 785) 2014-07-01 16:39:41,527.527 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1415 16 6131 6115 17.283 28 0.323238 3.69851 2014-07-01 16:39:42,527.527 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1416 16 6131 6115 17.2708 0 - 3.69851 2014-07-01 16:39:43,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:1708: expect (ObjNum 525 snap 0 seq_num 525) 2014-07-01 16:39:43,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1417 16 6131 6115 17.2586 0 - 3.69851 2014-07-01 16:39:44,527.527 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1418 16 6131 6115 17.2464 0 - 3.69851 2014-07-01 16:39:44,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: finishing write tid 3 to mira0435601-153 2014-07-01 16:39:44,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: finishing write tid 5 to mira0435601-325 2014-07-01 16:39:44,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 126 (ObjNum 1154 snap 0 seq_num 1154) dirty exists 2014-07-01 16:39:44,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: finishing write tid 4 to mira0435601-425 2014-07-01 16:39:44,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 115 (ObjNum 1146 snap 0 seq_num 1146) dirty exists 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 1 to mira0435601-179 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 2 to mira0435601-179 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 3 to mira0435601-179 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: done (15 left) 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: done (14 left) 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: done (13 left) 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: done (12 left) 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: done (11 left) 2014-07-01 16:39:44,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: done (10 left) 2014-07-01 16:39:44,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: done (9 left) 2014-07-01 16:39:44,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: done (8 left) 2014-07-01 16:39:44,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1708: done (7 left) 2014-07-01 16:39:45,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1710: done (6 left) 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1714: done (5 left) 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1715: read oid 15 snap -1 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 4 to mira0435601-179 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 6 to mira0435601-179 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 358 current snap is 0 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: seq_num 1162 ranges {475136=507904,983040=540672,1523712=606208,2129920=458752} 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: writing mira0435601-358 from 475136 to 983040 tid 1 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: writing mira0435601-358 from 983040 to 1523712 tid 2 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: writing mira0435601-358 from 1523712 to 2129920 tid 3 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: writing mira0435601-358 from 2129920 to 2588672 tid 4 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: seq_num 1163 ranges {5439488=532480,5971968=401408} 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: writing mira0435601-21 from 5439488 to 5971968 tid 1 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: writing mira0435601-21 from 5971968 to 6373376 tid 2 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 374 current snap is 0 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: seq_num 1164 ranges {2158592=454656,2613248=753664,3366912=450560} 2014-07-01 16:39:45,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: writing mira0435601-374 from 2158592 to 2613248 tid 1 2014-07-01 16:39:45,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: writing mira0435601-374 from 2613248 to 3366912 tid 2 2014-07-01 16:39:45,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: writing mira0435601-374 from 3366912 to 3817472 tid 3 2014-07-01 16:39:45,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1719: delete oid 117 current snap is 0 2014-07-01 16:39:45,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1419 16 6131 6115 17.2343 0 - 3.69851 2014-07-01 16:39:45,775.775 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:46,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:39:46.440675min lat: 0.057269 max lat: 294.525 avg lat: 3.69851 2014-07-01 16:39:46,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:39:46,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1420 16 6131 6115 17.2221 0 - 3.69851 2014-07-01 16:39:46,543.543 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:47,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 1 to mira0435601-358 2014-07-01 16:39:47,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1715: expect (ObjNum 843 snap 0 seq_num 843) 2014-07-01 16:39:47,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1421 16 6143 6127 17.2438 8 6.97369 3.70666 2014-07-01 16:39:48,097.097 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:48,140.140 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_resize_down ... ok 2014-07-01 16:39:48,527.527 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1422 16 6154 6138 17.2626 44 0.976609 3.70491 2014-07-01 16:39:48,794.794 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:39:41.691054', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883176824, u'health': u'HEALTH_OK', u'kb_used': 13604228, u'store_stats': {u'bytes_total': 10792982, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:38:54.534824', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886732856, u'health': u'HEALTH_OK', u'kb_used': 10048196, u'store_stats': {u'bytes_total': 11841004, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452380}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2829/13591 objects degraded (20.815%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 9738623, u'degraded_objects': 2829, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 26690295718, u'degraded_total': 13591, u'bytes_used': 50272317440, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 16047737, u'op_per_sec': 10, u'version': 1225, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3736, u'degraded_ratio': u'20.815', u'bytes_avail': 4948310220800}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 569, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:48,794.794 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: finishing write tid 1 to mira0435601-21 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: finishing write tid 2 to mira0435601-21 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 2 to mira0435601-358 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 3 to mira0435601-358 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 4 to mira0435601-358 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: finishing write tid 4 to mira0435601-21 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 6 to mira0435601-358 2014-07-01 16:39:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: finishing write tid 1 to mira0435601-374 2014-07-01 16:39:49,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: finishing write tid 2 to mira0435601-374 2014-07-01 16:39:49,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: finishing write tid 3 to mira0435601-374 2014-07-01 16:39:49,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: finishing write tid 5 to mira0435601-374 2014-07-01 16:39:49,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1423 16 6154 6138 17.2505 0 - 3.70491 2014-07-01 16:39:49,567.567 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:39:49,568.568 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:50,528.528 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1424 16 6154 6138 17.2383 0 - 3.70491 2014-07-01 16:39:51,529.529 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1425 16 6154 6138 17.2262 0 - 3.70491 2014-07-01 16:39:52,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1715: done (9 left) 2014-07-01 16:39:52,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1719: done (8 left) 2014-07-01 16:39:52,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1720: read oid 297 snap -1 2014-07-01 16:39:52,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1721: delete oid 93 current snap is 0 2014-07-01 16:39:52,274.274 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:52,529.529 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1426 16 6166 6150 17.2478 12 0.152569 3.70404 2014-07-01 16:39:53,529.529 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1427 16 6173 6157 17.2553 28 0.88893 3.70287 2014-07-01 16:39:54,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1428 16 6177 6161 17.2545 16 0.093244 3.70053 2014-07-01 16:39:55,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1429 16 6177 6161 17.2424 0 - 3.70053 2014-07-01 16:39:56,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1430 16 6177 6161 17.2303 0 - 3.70053 2014-07-01 16:39:56,761.761 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:39:57,532.532 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:39:57,540.540 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1431 16 6177 6161 17.2183 0 - 3.70053 2014-07-01 16:39:58,530.530 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1432 16 6177 6161 17.2063 0 - 3.70053 2014-07-01 16:39:59,090.090 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:59,301.301 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_rollback_to_snap ... ok 2014-07-01 16:39:59,531.531 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1433 16 6178 6162 17.197 0.8 6.67666 3.70101 2014-07-01 16:39:59,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 5 to mira0435601-262 2014-07-01 16:39:59,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 236 (ObjNum 1159 snap 0 seq_num 1159) dirty exists 2014-07-01 16:39:59,802.802 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:39:41.691054', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883176824, u'health': u'HEALTH_OK', u'kb_used': 13604228, u'store_stats': {u'bytes_total': 10792982, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:39:54.535210', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886741232, u'health': u'HEALTH_OK', u'kb_used': 10039820, u'store_stats': {u'bytes_total': 3890753, u'bytes_log': 65536, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3759665}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2828/13673 objects degraded (20.683%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2828, u'num_pgs': 244, u'data_bytes': 26870802598, u'degraded_total': 13673, u'bytes_used': 50781564928, u'op_per_sec': 8, u'write_bytes_sec': 20753100, u'version': 1233, u'pgs_by_state': [{u'count': 30, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2026, u'degraded_ratio': u'20.683', u'bytes_avail': 4947800973312}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 571, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:39:59,802.802 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:39:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: done (9 left) 2014-07-01 16:39:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1721: done (8 left) 2014-07-01 16:39:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1722: read oid 370 snap -1 2014-07-01 16:39:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:1723: read oid 216 snap -1 2014-07-01 16:39:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: read oid 51 snap -1 2014-07-01 16:39:59,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:1725: delete oid 139 current snap is 0 2014-07-01 16:40:00,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: finishing write tid 6 to mira0435601-374 2014-07-01 16:40:00,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 101 (ObjNum 1164 snap 0 seq_num 1164) dirty exists 2014-07-01 16:40:00,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1723: expect (ObjNum 215 snap 0 seq_num 215) 2014-07-01 16:40:00,531.531 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1434 16 6188 6172 17.2129 40 0.12554 3.70435 2014-07-01 16:40:00,599.599 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:40:00,599.599 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:01,350.350 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:01,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1435 16 6190 6174 17.2065 8 0.168534 3.70319 2014-07-01 16:40:02,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1436 16 6190 6174 17.1945 0 - 3.70319 2014-07-01 16:40:03,531.531 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1437 16 6190 6174 17.1826 0 - 3.70319 2014-07-01 16:40:04,531.531 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1438 16 6190 6174 17.1706 0 - 3.70319 2014-07-01 16:40:05,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1439 16 6190 6174 17.1587 0 - 3.70319 2014-07-01 16:40:05,736.736 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:06,499.499 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:06,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: finishing write tid 7 to mira0435601-358 2014-07-01 16:40:06,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 197 (ObjNum 1162 snap 0 seq_num 1162) dirty exists 2014-07-01 16:40:06,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:1720: expect (ObjNum 296 snap 0 seq_num 296) 2014-07-01 16:40:06,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:40:06.444856min lat: 0.057269 max lat: 294.525 avg lat: 3.71948 2014-07-01 16:40:06,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:40:06,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1440 16 6199 6183 17.1718 7.2 18.212 3.71948 2014-07-01 16:40:07,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1441 16 6217 6201 17.2098 72 0.700991 3.71747 2014-07-01 16:40:08,078.078 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: finishing write tid 4 to mira0435601-153 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 126 (ObjNum 1157 snap 0 seq_num 1157) dirty exists 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: finishing write tid 4 to mira0435601-332 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 117 (ObjNum 1158 snap 0 seq_num 1158) dirty exists 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: done (11 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: done (10 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: done (9 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: done (8 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1720: done (7 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1723: done (6 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1725: done (5 left) 2014-07-01 16:40:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1726: read oid 305 snap -1 2014-07-01 16:40:08,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: finishing write tid 4 to mira0435601-57 2014-07-01 16:40:08,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 118 (ObjNum 1155 snap 0 seq_num 1155) dirty exists 2014-07-01 16:40:08,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: done (5 left) 2014-07-01 16:40:08,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1727: read oid 222 snap -1 2014-07-01 16:40:08,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: expect (ObjNum 50 snap 0 seq_num 50) 2014-07-01 16:40:08,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1442 16 6223 6207 17.2145 24 1.04803 3.71471 2014-07-01 16:40:08,815.815 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 12, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 12, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.011722', u'skew': u'-0.003366', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.002462', u'skew': u'-0.012915', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:39:41.691054', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883176824, u'health': u'HEALTH_OK', u'kb_used': 13604228, u'store_stats': {u'bytes_total': 10792982, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452934}}, {u'last_updated': u'2014-07-01 07:39:54.535210', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886741232, u'health': u'HEALTH_OK', u'kb_used': 10039820, u'store_stats': {u'bytes_total': 3890753, u'bytes_log': 65536, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3759665}}, {u'last_updated': u'2014-07-01 07:39:11.317732', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886728928, u'health': u'HEALTH_OK', u'kb_used': 10052124, u'store_stats': {u'bytes_total': 12889599, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3452399}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'8 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2828/13717 objects degraded (20.617%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2828, u'num_pgs': 244, u'data_bytes': 26971465526, u'degraded_total': 13717, u'bytes_used': 51111075840, u'op_per_sec': 7, u'write_bytes_sec': 13566406, u'version': 1240, u'pgs_by_state': [{u'count': 28, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1578952, u'degraded_ratio': u'20.617', u'bytes_avail': 4947471462400}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 572, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:40:08,816.816 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: finishing write tid 7 to mira0435601-179 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 179 v 202 (ObjNum 1161 snap 0 seq_num 1161) dirty exists 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: finishing write tid 5 to mira0435601-21 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 170 (ObjNum 1163 snap 0 seq_num 1163) dirty exists 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: done (5 left) 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: done (4 left) 2014-07-01 16:40:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: done (3 left) 2014-07-01 16:40:09,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1726: done (2 left) 2014-07-01 16:40:09,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 413 current snap is 0 2014-07-01 16:40:09,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: seq_num 1168 ranges {196608=548864,745472=401408} 2014-07-01 16:40:09,516.516 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:40:09,517.517 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:09,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: writing mira0435601-413 from 196608 to 745472 tid 1 2014-07-01 16:40:09,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1443 16 6224 6208 17.2054 4 1.61698 3.71438 2014-07-01 16:40:09,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: writing mira0435601-413 from 745472 to 1146880 tid 2 2014-07-01 16:40:09,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: read oid 44 snap -1 2014-07-01 16:40:09,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: delete oid 101 current snap is 0 2014-07-01 16:40:10,308.308 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:10,532.532 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1444 16 6224 6208 17.1934 0 - 3.71438 2014-07-01 16:40:11,533.533 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1445 16 6224 6208 17.1815 0 - 3.71438 2014-07-01 16:40:12,533.533 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1446 16 6224 6208 17.1697 0 - 3.71438 2014-07-01 16:40:13,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1447 16 6224 6208 17.1578 0 - 3.71438 2014-07-01 16:40:14,533.533 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1448 16 6224 6208 17.1459 0 - 3.71438 2014-07-01 16:40:14,835.835 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:15,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1449 16 6224 6208 17.1341 0 - 3.71438 2014-07-01 16:40:16,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1450 16 6224 6208 17.1223 0 - 3.71438 2014-07-01 16:40:17,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1451 16 6224 6208 17.1105 0 - 3.71438 2014-07-01 16:40:18,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1452 16 6227 6211 17.107 1.33333 11.2871 3.71792 2014-07-01 16:40:19,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1453 16 6227 6211 17.0952 0 - 3.71792 2014-07-01 16:40:19,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1722: expect (ObjNum 369 snap 0 seq_num 369) 2014-07-01 16:40:19,894.894 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:20,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: expect (ObjNum 590 snap 0 seq_num 590) 2014-07-01 16:40:20,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1454 16 6233 6217 17.0999 12 0.189823 3.71856 2014-07-01 16:40:21,534.534 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1455 16 6235 6219 17.0937 8 2.79379 3.71826 2014-07-01 16:40:21,640.640 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: finishing write tid 1 to mira0435601-413 2014-07-01 16:40:22,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: finishing write tid 2 to mira0435601-413 2014-07-01 16:40:22,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: finishing write tid 4 to mira0435601-413 2014-07-01 16:40:22,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: finishing write tid 5 to mira0435601-413 2014-07-01 16:40:22,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 205 (ObjNum 1168 snap 0 seq_num 1168) dirty exists 2014-07-01 16:40:22,345.345 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2808/13744 objects degraded (20.431%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 69996405, u'degraded_objects': 2808, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 27029362620, u'degraded_total': 13744, u'bytes_used': 51332104192, u'recovering_objects_per_sec': 16, u'write_bytes_sec': 16471752, u'op_per_sec': 11, u'version': 1244, u'pgs_by_state': [{u'count': 28, u'state_name': u'active+recovery_wait'}, {u'count': 193, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 4020, u'degraded_ratio': u'20.431', u'bytes_avail': 4947250434048}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 574, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:40:22,345.345 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:22,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1456 16 6238 6222 17.0902 12 0.682364 3.71893 2014-07-01 16:40:23,112.112 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:40:23,112.112 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:23,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1457 16 6238 6222 17.0785 0 - 3.71893 2014-07-01 16:40:23,945.945 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:24,535.535 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1458 16 6238 6222 17.0667 0 - 3.71893 2014-07-01 16:40:25,536.536 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1459 16 6238 6222 17.055 0 - 3.71893 2014-07-01 16:40:25,689.689 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_rollback_to_snap_sparse ... ok 2014-07-01 16:40:26,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:40:26.448803min lat: 0.057269 max lat: 294.525 avg lat: 3.72884 2014-07-01 16:40:26,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:40:26,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1460 16 6257 6241 17.0954 19 0.498365 3.72884 2014-07-01 16:40:27,536.536 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1461 16 6262 6246 17.0974 20 0.10923 3.72608 2014-07-01 16:40:28,331.331 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:28,536.536 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1462 16 6262 6246 17.0857 0 - 3.72608 2014-07-01 16:40:29,094.094 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:29,536.536 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1463 16 6262 6246 17.074 0 - 3.72608 2014-07-01 16:40:30,537.537 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1464 16 6262 6246 17.0624 0 - 3.72608 2014-07-01 16:40:30,647.647 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:31,343.343 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'4 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'54 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2753/13684 objects degraded (20.118%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2753, u'num_pgs': 244, u'data_bytes': 26907727804, u'degraded_total': 13684, u'bytes_used': 51452825600, u'version': 1254, u'pgs_by_state': [{u'count': 3, u'state_name': u'peering'}, {u'count': 28, u'state_name': u'active+recovery_wait'}, {u'count': 190, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'remapped+peering'}, {u'count': 22, u'state_name': u'active+recovering'}], u'degraded_ratio': u'20.118', u'bytes_avail': 4947129712640}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 580, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:40:31,343.343 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:31,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:1727: expect (ObjNum 1019 snap 0 seq_num 1019) 2014-07-01 16:40:31,537.537 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1465 16 6271 6255 17.0753 9 5.68083 3.7421 2014-07-01 16:40:32,108.108 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:40:32,108.108 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:32,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1466 16 6279 6263 17.0855 32 0.107075 3.73787 2014-07-01 16:40:32,920.920 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:33,537.537 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1467 16 6280 6264 17.0765 4 0.119266 3.7373 2014-07-01 16:40:34,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1468 16 6280 6264 17.0649 0 - 3.7373 2014-07-01 16:40:35,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1469 16 6280 6264 17.0533 0 - 3.7373 2014-07-01 16:40:36,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1470 16 6280 6264 17.0417 0 - 3.7373 2014-07-01 16:40:37,304.304 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:37,538.538 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1471 16 6280 6264 17.0301 0 - 3.7373 2014-07-01 16:40:38,160.160 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:38,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1472 16 6280 6264 17.0185 0 - 3.7373 2014-07-01 16:40:39,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1473 16 6280 6264 17.007 0 - 3.7373 2014-07-01 16:40:39,720.720 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:40,436.436 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'51 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2560/13873 objects degraded (18.453%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 81914385, u'degraded_objects': 2560, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 27302463698, u'degraded_total': 13873, u'bytes_used': 51795025920, u'recovering_objects_per_sec': 21, u'write_bytes_sec': 121733920, u'op_per_sec': 61, u'version': 1261, u'pgs_by_state': [{u'count': 4, u'state_name': u'inactive'}, {u'count': 5, u'state_name': u'peering'}, {u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 189, u'state_name': u'active+clean'}, {u'count': 5, u'state_name': u'remapped+peering'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 7049328, u'degraded_ratio': u'18.453', u'bytes_avail': 4946787512320}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 582, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:40:40,436.436 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:40,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1474 16 6280 6264 16.9954 0 - 3.7373 2014-07-01 16:40:41,197.197 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:40:41,197.197 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:41,539.539 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1475 16 6280 6264 16.9839 0 - 3.7373 2014-07-01 16:40:42,084.084 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:42,540.540 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1476 16 6301 6285 17.0293 9.33333 0.11796 3.74242 2014-07-01 16:40:43,540.540 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1477 16 6313 6297 17.0503 48 0.059448 3.7355 2014-07-01 16:40:44,540.540 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1478 16 6329 6313 17.082 64 0.186241 3.74408 2014-07-01 16:40:45,540.540 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1479 16 6329 6313 17.0705 0 - 3.74408 2014-07-01 16:40:46,500.500 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:46,613.613 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:40:46.453195min lat: 0.057269 max lat: 294.525 avg lat: 3.74303 2014-07-01 16:40:46,613.613 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:40:46,613.613 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1480 16 6332 6316 17.0671 6 0.09479 3.74303 2014-07-01 16:40:47,316.316 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:47,613.613 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1481 16 6341 6325 17.0798 36 0.209806 3.73953 2014-07-01 16:40:48,612.612 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1482 16 6341 6325 17.0683 0 - 3.73953 2014-07-01 16:40:49,609.609 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1483 16 6341 6325 17.0568 0 - 3.73953 2014-07-01 16:40:50,611.611 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1484 16 6350 6334 17.0696 12 6.7693 3.74256 2014-07-01 16:40:51,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1485 16 6350 6334 17.0581 0 - 3.74256 2014-07-01 16:40:51,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1722: done (4 left) 2014-07-01 16:40:51,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1727: done (3 left) 2014-07-01 16:40:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: done (2 left) 2014-07-01 16:40:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: done (1 left) 2014-07-01 16:40:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: done (0 left) 2014-07-01 16:40:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: read oid 462 snap -1 2014-07-01 16:40:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:1732: delete oid 167 current snap is 0 2014-07-01 16:40:52,542.542 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1486 16 6358 6342 17.0681 16 5.15492 3.74552 2014-07-01 16:40:53,542.542 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1487 16 6362 6346 17.0674 16 0.938276 3.74364 2014-07-01 16:40:54,542.542 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1488 16 6362 6346 17.0559 0 - 3.74364 2014-07-01 16:40:55,542.542 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1489 16 6362 6346 17.0445 0 - 3.74364 2014-07-01 16:40:56,469.469 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:56,543.543 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1490 16 6362 6346 17.033 0 - 3.74364 2014-07-01 16:40:57,148.148 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:40:41.692636', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883143896, u'health': u'HEALTH_OK', u'kb_used': 13637156, u'store_stats': {u'bytes_total': 4489332, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'50 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2666/14016 objects degraded (19.021%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2666, u'num_pgs': 244, u'data_bytes': 27600578748, u'degraded_total': 14016, u'bytes_used': 52607971328, u'op_per_sec': 0, u'write_bytes_sec': 1295866, u'version': 1271, u'pgs_by_state': [{u'count': 6, u'state_name': u'active'}, {u'count': 21, u'state_name': u'active+recovery_wait'}, {u'count': 194, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'19.021', u'bytes_avail': 4945974566912}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 584, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:40:57,148.148 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:40:57,543.543 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1491 16 6362 6346 17.0216 0 - 3.74364 2014-07-01 16:40:57,889.889 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:40:57,890.890 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:40:58,543.543 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1492 16 6362 6346 17.0102 0 - 3.74364 2014-07-01 16:40:58,716.716 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:40:59,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1493 16 6362 6346 16.9988 0 - 3.74364 2014-07-01 16:41:00,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1494 16 6362 6346 16.9874 0 - 3.74364 2014-07-01 16:41:01,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1495 16 6362 6346 16.9761 0 - 3.74364 2014-07-01 16:41:01,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: expect (ObjNum 461 snap 0 seq_num 461) 2014-07-01 16:41:02,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1496 16 6375 6359 16.9995 5.77778 0.513581 3.75413 2014-07-01 16:41:02,708.708 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_rollback_with_resize ... ok 2014-07-01 16:41:03,317.317 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:03,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1497 16 6375 6359 16.9881 0 - 3.75413 2014-07-01 16:41:03,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: done (1 left) 2014-07-01 16:41:03,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1732: done (0 left) 2014-07-01 16:41:03,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:41:03,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: seq_num 1171 ranges {401408=475136,876544=401408} 2014-07-01 16:41:03,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0435601-284 from 401408 to 876544 tid 1 2014-07-01 16:41:03,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0435601-284 from 876544 to 1277952 tid 2 2014-07-01 16:41:03,898.898 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1734: read oid 286 snap -1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: read oid 125 snap -1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: read oid 337 snap -1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1737: read oid 33 snap -1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: read oid 129 snap -1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: seq_num 1172 ranges {0=745472} 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: writing mira0435601-445 from 0 to 745472 tid 1 2014-07-01 16:41:03,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1740: read oid 23 snap -1 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1741: read oid 485 snap -1 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 459 current snap is 0 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: seq_num 1173 ranges {2801664=667648,3469312=790528,4259840=12288} 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: writing mira0435601-459 from 2801664 to 3469312 tid 1 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: writing mira0435601-459 from 3469312 to 4259840 tid 2 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: writing mira0435601-459 from 4259840 to 4272128 tid 3 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 306 current snap is 0 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: seq_num 1174 ranges {2023424=294912} 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: writing mira0435601-306 from 2023424 to 2318336 tid 1 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: read oid 402 snap -1 2014-07-01 16:41:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: expect (ObjNum 124 snap 0 seq_num 124) 2014-07-01 16:41:04,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:1737: expect (ObjNum 1127 snap 0 seq_num 1127) 2014-07-01 16:41:04,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1498 16 6375 6359 16.9768 0 - 3.75413 2014-07-01 16:41:04,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1741: expect (ObjNum 939 snap 0 seq_num 939) 2014-07-01 16:41:05,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: done (11 left) 2014-07-01 16:41:05,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1737: done (10 left) 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:1741: done (9 left) 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 233 current snap is 0 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: seq_num 1175 ranges {2891776=163840} 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: writing mira0435601-233 from 2891776 to 3055616 tid 1 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 155 current snap is 0 2014-07-01 16:41:05,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: seq_num 1176 ranges {0=36864} 2014-07-01 16:41:05,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: writing mira0435601-155 from 0 to 36864 tid 1 2014-07-01 16:41:05,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: delete oid 248 current snap is 0 2014-07-01 16:41:05,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: finishing write tid 1 to mira0435601-306 2014-07-01 16:41:05,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: finishing write tid 3 to mira0435601-306 2014-07-01 16:41:05,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1734: expect (ObjNum 736 snap 0 seq_num 736) 2014-07-01 16:41:05,544.544 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1499 16 6386 6370 16.9948 14.6667 0.238964 3.76094 2014-07-01 16:41:05,934.934 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:06,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:41:06.457289min lat: 0.057269 max lat: 294.525 avg lat: 3.75931 2014-07-01 16:41:06,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:41:06,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1500 16 6389 6373 16.9915 12 0.192543 3.75931 2014-07-01 16:41:06,738.738 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:40:41.692636', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883143896, u'health': u'HEALTH_OK', u'kb_used': 13637156, u'store_stats': {u'bytes_total': 4489332, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'27 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'50 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2675/14052 objects degraded (19.036%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12404057, u'degraded_objects': 2675, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 27684759462, u'degraded_total': 14052, u'bytes_used': 52743106560, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 8560092, u'op_per_sec': 10, u'version': 1277, u'pgs_by_state': [{u'count': 27, u'state_name': u'active+recovery_wait'}, {u'count': 194, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2632632, u'degraded_ratio': u'19.036', u'bytes_avail': 4945839431680}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 586, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:06,738.738 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:06,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: finishing write tid 4 to mira0435601-306 2014-07-01 16:41:06,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 228 (ObjNum 1174 snap 0 seq_num 1174) dirty exists 2014-07-01 16:41:06,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: expect (ObjNum 1079 snap 0 seq_num 1079) 2014-07-01 16:41:07,173.173 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_set_no_snap ... ok 2014-07-01 16:41:07,555.555 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:41:07,556.556 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:07,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1501 16 6402 6386 17.0148 52 0.385025 3.75875 2014-07-01 16:41:08,373.373 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:08,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1502 16 6403 6387 17.0061 4 0.244884 3.7582 2014-07-01 16:41:09,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1503 16 6403 6387 16.9948 0 - 3.7582 2014-07-01 16:41:10,619.619 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1504 16 6403 6387 16.9835 0 - 3.7582 2014-07-01 16:41:11,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1505 16 6403 6387 16.9722 0 - 3.7582 2014-07-01 16:41:12,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1506 16 6403 6387 16.9609 0 - 3.7582 2014-07-01 16:41:13,216.216 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:13,545.545 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1507 16 6408 6392 16.963 4 0.059137 3.7587 2014-07-01 16:41:14,197.197 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:14,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 1 to mira0435601-284 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 2 to mira0435601-284 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 4 to mira0435601-284 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: finishing write tid 1 to mira0435601-459 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: finishing write tid 2 to mira0435601-459 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: finishing write tid 3 to mira0435601-459 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 1 to mira0435601-445 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 3 to mira0435601-445 2014-07-01 16:41:14,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: expect (ObjNum 401 snap 0 seq_num 401) 2014-07-01 16:41:14,554.554 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1508 16 6416 6400 16.9729 32 7.38633 3.76412 2014-07-01 16:41:14,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: finishing write tid 5 to mira0435601-459 2014-07-01 16:41:14,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: finishing write tid 1 to mira0435601-233 2014-07-01 16:41:14,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 5 to mira0435601-284 2014-07-01 16:41:14,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 239 (ObjNum 1171 snap 0 seq_num 1171) dirty exists 2014-07-01 16:41:14,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: finishing write tid 6 to mira0435601-459 2014-07-01 16:41:14,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 238 (ObjNum 1173 snap 0 seq_num 1173) dirty exists 2014-07-01 16:41:14,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: expect (ObjNum 929 snap 0 seq_num 929) 2014-07-01 16:41:15,546.546 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1509 16 6421 6405 16.9749 20 0.169811 3.76151 2014-07-01 16:41:15,947.947 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:16,547.547 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1510 16 6421 6405 16.9637 0 - 3.76151 2014-07-01 16:41:16,662.662 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:40:41.692636', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883143896, u'health': u'HEALTH_OK', u'kb_used': 13637156, u'store_stats': {u'bytes_total': 4489332, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'27 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'50 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2689/14116 objects degraded (19.049%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 6258087, u'degraded_objects': 2689, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 27813881643, u'degraded_total': 14116, u'bytes_used': 53136896000, u'recovering_objects_per_sec': 1, u'version': 1285, u'pgs_by_state': [{u'count': 27, u'state_name': u'active+recovery_wait'}, {u'count': 194, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'19.049', u'bytes_avail': 4945445642240}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 588, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:16,663.663 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:17,429.429 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:41:17,429.429 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:17,546.546 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1511 16 6429 6413 16.9736 16 0.191109 3.75861 2014-07-01 16:41:18,222.222 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:18,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: finishing write tid 3 to mira0435601-233 2014-07-01 16:41:18,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1740: expect (ObjNum 22 snap 0 seq_num 22) 2014-07-01 16:41:18,547.547 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1512 16 6430 6414 16.9651 4 1.1091 3.7582 2014-07-01 16:41:19,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: finishing write tid 1 to mira0435601-155 2014-07-01 16:41:19,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: finishing write tid 3 to mira0435601-155 2014-07-01 16:41:19,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 4 to mira0435601-445 2014-07-01 16:41:19,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 184 (ObjNum 1172 snap 0 seq_num 1172) dirty exists 2014-07-01 16:41:19,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: finishing write tid 4 to mira0435601-233 2014-07-01 16:41:19,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 126 (ObjNum 1175 snap 0 seq_num 1175) dirty exists 2014-07-01 16:41:19,547.547 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1513 16 6430 6414 16.9538 0 - 3.7582 2014-07-01 16:41:20,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1514 16 6430 6414 16.9426 0 - 3.7582 2014-07-01 16:41:20,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: finishing write tid 4 to mira0435601-155 2014-07-01 16:41:20,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 209 (ObjNum 1176 snap 0 seq_num 1176) dirty exists 2014-07-01 16:41:20,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: done (11 left) 2014-07-01 16:41:20,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1734: done (10 left) 2014-07-01 16:41:20,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: done (9 left) 2014-07-01 16:41:20,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: done (8 left) 2014-07-01 16:41:20,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: done (7 left) 2014-07-01 16:41:20,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:1740: done (6 left) 2014-07-01 16:41:20,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: done (5 left) 2014-07-01 16:41:20,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: done (4 left) 2014-07-01 16:41:20,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: done (3 left) 2014-07-01 16:41:20,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: done (2 left) 2014-07-01 16:41:20,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: done (1 left) 2014-07-01 16:41:20,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: done (0 left) 2014-07-01 16:41:20,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: read oid 409 snap -1 2014-07-01 16:41:21,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1749: delete oid 134 current snap is 0 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1749: done (1 left) 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1750: read oid 497 snap -1 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: seq_num 1179 ranges {2084864=659456,2744320=536576,3280896=667648} 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: writing mira0435601-9 from 2084864 to 2744320 tid 1 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: writing mira0435601-9 from 2744320 to 3280896 tid 2 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: writing mira0435601-9 from 3280896 to 3948544 tid 3 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1752: delete oid 239 current snap is 0 2014-07-01 16:41:21,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1750: expect (ObjNum 496 snap 0 seq_num 496) 2014-07-01 16:41:21,455.455 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_set_snap ... ok 2014-07-01 16:41:21,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1515 16 6448 6432 16.979 24 0.54856 3.76495 2014-07-01 16:41:22,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: finishing write tid 1 to mira0435601-9 2014-07-01 16:41:22,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: finishing write tid 2 to mira0435601-9 2014-07-01 16:41:22,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: finishing write tid 3 to mira0435601-9 2014-07-01 16:41:22,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: finishing write tid 5 to mira0435601-9 2014-07-01 16:41:22,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: finishing write tid 6 to mira0435601-9 2014-07-01 16:41:22,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 297 (ObjNum 1179 snap 0 seq_num 1179) dirty exists 2014-07-01 16:41:22,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1750: done (3 left) 2014-07-01 16:41:22,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: done (2 left) 2014-07-01 16:41:22,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1752: done (1 left) 2014-07-01 16:41:22,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: read oid 149 snap -1 2014-07-01 16:41:22,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 392 current snap is 0 2014-07-01 16:41:22,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: seq_num 1181 ranges {1155072=557056} 2014-07-01 16:41:22,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: writing mira0435601-392 from 1155072 to 1712128 tid 1 2014-07-01 16:41:22,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1755: read oid 268 snap -1 2014-07-01 16:41:22,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 15 current snap is 0 2014-07-01 16:41:22,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: seq_num 1182 ranges {2789376=479232,3268608=737280,4005888=65536} 2014-07-01 16:41:22,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: writing mira0435601-15 from 2789376 to 3268608 tid 1 2014-07-01 16:41:22,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: writing mira0435601-15 from 3268608 to 4005888 tid 2 2014-07-01 16:41:22,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: writing mira0435601-15 from 4005888 to 4071424 tid 3 2014-07-01 16:41:22,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 272 current snap is 0 2014-07-01 16:41:22,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: seq_num 1183 ranges {966656=102400} 2014-07-01 16:41:22,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: writing mira0435601-272 from 966656 to 1069056 tid 1 2014-07-01 16:41:22,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1758: read oid 464 snap -1 2014-07-01 16:41:22,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1759: delete oid 382 current snap is 0 2014-07-01 16:41:22,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1516 16 6454 6438 16.9836 24 0.17313 3.76174 2014-07-01 16:41:22,784.784 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:23,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1758: expect (ObjNum 1045 snap 0 seq_num 1045) 2014-07-01 16:41:23,384.384 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:23,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1517 16 6454 6438 16.9724 0 - 3.76174 2014-07-01 16:41:24,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1518 16 6469 6453 17.0007 30 0.56929 3.76044 2014-07-01 16:41:25,144.144 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:25,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1519 16 6473 6457 17.0001 16 0.292498 3.75834 2014-07-01 16:41:25,852.852 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:40:41.692636', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883143896, u'health': u'HEALTH_OK', u'kb_used': 13637156, u'store_stats': {u'bytes_total': 4489332, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'27 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'49 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2623/14199 objects degraded (18.473%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2623, u'num_pgs': 244, u'data_bytes': 27994437419, u'degraded_total': 14199, u'bytes_used': 53450465280, u'op_per_sec': 8, u'write_bytes_sec': 20069910, u'version': 1290, u'pgs_by_state': [{u'count': 27, u'state_name': u'active+recovery_wait'}, {u'count': 195, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 557337, u'degraded_ratio': u'18.473', u'bytes_avail': 4945132072960}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 589, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:25,853.853 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:26,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:41:26.460382min lat: 0.057269 max lat: 294.525 avg lat: 3.75834 2014-07-01 16:41:26,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:41:26,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1520 16 6473 6457 16.9889 0 - 3.75834 2014-07-01 16:41:26,636.636 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:41:26,636.636 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:27,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: expect (ObjNum 700 snap 0 seq_num 700) 2014-07-01 16:41:27,394.394 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:27,548.548 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1521 16 6473 6457 16.9777 0 - 3.75834 2014-07-01 16:41:28,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1522 16 6473 6457 16.9666 0 - 3.75834 2014-07-01 16:41:29,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1523 16 6473 6457 16.9554 0 - 3.75834 2014-07-01 16:41:30,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: expect (ObjNum 1102 snap 0 seq_num 1102) 2014-07-01 16:41:30,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1524 16 6491 6475 16.9916 14.4 0.094122 3.75989 2014-07-01 16:41:31,550.550 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1525 16 6491 6475 16.9804 0 - 3.75989 2014-07-01 16:41:31,958.958 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:32,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1755: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:41:32,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1526 16 6491 6475 16.9693 0 - 3.75989 2014-07-01 16:41:32,730.730 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:33,550.550 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1527 16 6491 6475 16.9582 0 - 3.75989 2014-07-01 16:41:33,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: finishing write tid 1 to mira0435601-392 2014-07-01 16:41:33,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: finishing write tid 3 to mira0435601-392 2014-07-01 16:41:33,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: finishing write tid 1 to mira0435601-15 2014-07-01 16:41:33,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: finishing write tid 2 to mira0435601-15 2014-07-01 16:41:33,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: finishing write tid 3 to mira0435601-15 2014-07-01 16:41:33,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: finishing write tid 5 to mira0435601-15 2014-07-01 16:41:33,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: done (7 left) 2014-07-01 16:41:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: done (6 left) 2014-07-01 16:41:33,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:1755: done (5 left) 2014-07-01 16:41:33,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1758: done (4 left) 2014-07-01 16:41:33,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1759: done (3 left) 2014-07-01 16:41:33,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: read oid 13 snap -1 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: seq_num 1185 ranges {6062080=651264,6713344=577536,7290880=659456,7950336=368640} 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: writing mira0435601-128 from 6062080 to 6713344 tid 1 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: writing mira0435601-128 from 6713344 to 7290880 tid 2 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: writing mira0435601-128 from 7290880 to 7950336 tid 3 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: writing mira0435601-128 from 7950336 to 8318976 tid 4 2014-07-01 16:41:33,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 232 current snap is 0 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: seq_num 1186 ranges {24576=724992,749568=561152,1310720=643072} 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: writing mira0435601-232 from 24576 to 749568 tid 1 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: writing mira0435601-232 from 749568 to 1310720 tid 2 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: writing mira0435601-232 from 1310720 to 1953792 tid 3 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 481 current snap is 0 2014-07-01 16:41:33,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: seq_num 1187 ranges {753664=598016,1351680=745472,2097152=458752} 2014-07-01 16:41:33,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: writing mira0435601-481 from 753664 to 1351680 tid 1 2014-07-01 16:41:33,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: writing mira0435601-481 from 1351680 to 2097152 tid 2 2014-07-01 16:41:33,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: writing mira0435601-481 from 2097152 to 2555904 tid 3 2014-07-01 16:41:34,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1764: delete oid 275 current snap is 0 2014-07-01 16:41:34,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: expect (ObjNum 817 snap 0 seq_num 817) 2014-07-01 16:41:34,307.307 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:34,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 1 to mira0435601-128 2014-07-01 16:41:34,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 2 to mira0435601-128 2014-07-01 16:41:34,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 3 to mira0435601-128 2014-07-01 16:41:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 4 to mira0435601-128 2014-07-01 16:41:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 6 to mira0435601-128 2014-07-01 16:41:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 7 to mira0435601-128 2014-07-01 16:41:34,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 167 (ObjNum 1185 snap 0 seq_num 1185) dirty exists 2014-07-01 16:41:34,549.549 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1528 16 6495 6479 16.9575 4 0.127513 3.76055 2014-07-01 16:41:35,043.043 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:40:41.692636', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883143896, u'health': u'HEALTH_OK', u'kb_used': 13637156, u'store_stats': {u'bytes_total': 4489332, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'49 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2633/14285 objects degraded (18.432%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 15340854, u'degraded_objects': 2633, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28179879846, u'degraded_total': 14285, u'bytes_used': 53921525760, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 35477088, u'op_per_sec': 11, u'version': 1296, u'pgs_by_state': [{u'count': 26, u'state_name': u'active+recovery_wait'}, {u'count': 195, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 867894, u'degraded_ratio': u'18.432', u'bytes_avail': 4944661012480}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 590, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:35,044.044 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:35,550.550 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1529 16 6495 6479 16.9465 0 - 3.76055 2014-07-01 16:41:35,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 1 to mira0435601-272 2014-07-01 16:41:35,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 3 to mira0435601-272 2014-07-01 16:41:35,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: finishing write tid 6 to mira0435601-15 2014-07-01 16:41:35,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 122 (ObjNum 1182 snap 0 seq_num 1182) dirty exists 2014-07-01 16:41:35,841.841 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:41:35,841.841 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:36,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: finishing write tid 4 to mira0435601-392 2014-07-01 16:41:36,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 392 v 211 (ObjNum 1181 snap 0 seq_num 1181) dirty exists 2014-07-01 16:41:36,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 4 to mira0435601-272 2014-07-01 16:41:36,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 119 (ObjNum 1183 snap 0 seq_num 1183) dirty exists 2014-07-01 16:41:36,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: finishing write tid 1 to mira0435601-481 2014-07-01 16:41:36,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: finishing write tid 2 to mira0435601-481 2014-07-01 16:41:36,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: finishing write tid 3 to mira0435601-481 2014-07-01 16:41:36,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: finishing write tid 5 to mira0435601-481 2014-07-01 16:41:36,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: finishing write tid 6 to mira0435601-481 2014-07-01 16:41:36,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 105 (ObjNum 1187 snap 0 seq_num 1187) dirty exists 2014-07-01 16:41:36,177.177 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_set_snap_deleted ... ok 2014-07-01 16:41:36,550.550 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1530 16 6506 6490 16.9641 22 0.085192 3.76428 2014-07-01 16:41:36,593.593 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:37,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1531 16 6506 6490 16.953 0 - 3.76428 2014-07-01 16:41:38,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1532 16 6506 6490 16.942 0 - 3.76428 2014-07-01 16:41:39,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1533 16 6506 6490 16.9309 0 - 3.76428 2014-07-01 16:41:40,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1534 16 6506 6490 16.9199 0 - 3.76428 2014-07-01 16:41:40,980.980 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:41,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1535 16 6506 6490 16.9089 0 - 3.76428 2014-07-01 16:41:41,734.734 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:42,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: done (7 left) 2014-07-01 16:41:42,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: done (6 left) 2014-07-01 16:41:42,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: done (5 left) 2014-07-01 16:41:42,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: done (4 left) 2014-07-01 16:41:42,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: done (3 left) 2014-07-01 16:41:42,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: done (2 left) 2014-07-01 16:41:42,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1764: done (1 left) 2014-07-01 16:41:42,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: read oid 340 snap -1 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: finishing write tid 1 to mira0435601-232 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1766: read oid 449 snap -1 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 198 current snap is 0 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: seq_num 1189 ranges {1572864=495616} 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: writing mira0435601-198 from 1572864 to 2068480 tid 1 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: read oid 47 snap -1 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: finishing write tid 2 to mira0435601-232 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: finishing write tid 3 to mira0435601-232 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: finishing write tid 5 to mira0435601-232 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: finishing write tid 6 to mira0435601-232 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 94 (ObjNum 1186 snap 0 seq_num 1186) dirty exists 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: done (4 left) 2014-07-01 16:41:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: read oid 460 snap -1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 40 current snap is 0 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: seq_num 1190 ranges {184320=368640} 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: writing mira0435601-40 from 184320 to 552960 tid 1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: read oid 295 snap -1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 192 current snap is 0 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: seq_num 1191 ranges {0=692224,692224=401408} 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: writing mira0435601-192 from 0 to 692224 tid 1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: writing mira0435601-192 from 692224 to 1093632 tid 2 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1773: read oid 172 snap -1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 459 current snap is 0 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: seq_num 1192 ranges {4272128=716800,4988928=708608,5697536=434176,6131712=462848} 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: writing mira0435601-459 from 4272128 to 4988928 tid 1 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: writing mira0435601-459 from 4988928 to 5697536 tid 2 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: writing mira0435601-459 from 5697536 to 6131712 tid 3 2014-07-01 16:41:42,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: writing mira0435601-459 from 6131712 to 6594560 tid 4 2014-07-01 16:41:42,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 247 current snap is 0 2014-07-01 16:41:42,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: seq_num 1193 ranges {1863680=589824,2453504=540672,2994176=552960,3547136=512000} 2014-07-01 16:41:42,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: writing mira0435601-247 from 1863680 to 2453504 tid 1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: writing mira0435601-247 from 2453504 to 2994176 tid 2 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: writing mira0435601-247 from 2994176 to 3547136 tid 3 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: writing mira0435601-247 from 3547136 to 4059136 tid 4 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1776: read oid 39 snap -1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1777: read oid 444 snap -1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1778: read oid 330 snap -1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 171 current snap is 0 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: seq_num 1194 ranges {778240=618496,1396736=401408} 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: writing mira0435601-171 from 778240 to 1396736 tid 1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: writing mira0435601-171 from 1396736 to 1798144 tid 2 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1780: read oid 356 snap -1 2014-07-01 16:41:42,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:42,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: expect (ObjNum 800 snap 0 seq_num 800) 2014-07-01 16:41:42,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1536 16 6522 6506 16.9395 10.6667 6.68821 3.77183 2014-07-01 16:41:43,307.307 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:43,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: expect (ObjNum 46 snap 0 seq_num 46) 2014-07-01 16:41:43,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:1778: expect (ObjNum 614 snap 0 seq_num 614) 2014-07-01 16:41:43,550.550 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1537 16 6530 6514 16.9493 32 0.285853 3.76972 2014-07-01 16:41:44,020.020 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'49 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2614/14321 objects degraded (18.253%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3707178, u'degraded_objects': 2614, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28263733035, u'degraded_total': 14321, u'bytes_used': 54243196928, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 8784438, u'op_per_sec': 3, u'version': 1300, u'pgs_by_state': [{u'count': 26, u'state_name': u'active+recovery_wait'}, {u'count': 195, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 641, u'degraded_ratio': u'18.253', u'bytes_avail': 4944339341312}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 590, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:44,020.020 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:44,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1776: expect (ObjNum 38 snap 0 seq_num 38) 2014-07-01 16:41:44,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1777: expect (ObjNum 802 snap 0 seq_num 802) 2014-07-01 16:41:44,551.551 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1538 16 6532 6516 16.9435 8 1.38048 3.76896 2014-07-01 16:41:44,823.823 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:41:44,823.823 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:45,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1766: expect (ObjNum 1081 snap 0 seq_num 1081) 2014-07-01 16:41:45,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: expect (ObjNum 915 snap 0 seq_num 915) 2014-07-01 16:41:45,547.547 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:45,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1539 16 6534 6518 16.9377 8 1.84391 3.7684 2014-07-01 16:41:46,552.552 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:41:46.463816min lat: 0.057269 max lat: 294.525 avg lat: 3.76781 2014-07-01 16:41:46,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:41:46,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1540 16 6537 6521 16.9345 12 1.126 3.76781 2014-07-01 16:41:46,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: finishing write tid 1 to mira0435601-40 2014-07-01 16:41:46,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: finishing write tid 3 to mira0435601-40 2014-07-01 16:41:46,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1773: expect (ObjNum 1087 snap 0 seq_num 1087) 2014-07-01 16:41:47,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1541 16 6539 6523 16.9287 8 1.38098 3.76705 2014-07-01 16:41:48,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1542 16 6539 6523 16.9177 0 - 3.76705 2014-07-01 16:41:49,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1543 16 6539 6523 16.9067 0 - 3.76705 2014-07-01 16:41:49,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: finishing write tid 1 to mira0435601-192 2014-07-01 16:41:49,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 1 to mira0435601-459 2014-07-01 16:41:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 2 to mira0435601-459 2014-07-01 16:41:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: finishing write tid 2 to mira0435601-192 2014-07-01 16:41:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: finishing write tid 4 to mira0435601-192 2014-07-01 16:41:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: finishing write tid 1 to mira0435601-171 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 3 to mira0435601-459 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 4 to mira0435601-459 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 6 to mira0435601-459 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: finishing write tid 2 to mira0435601-171 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: finishing write tid 4 to mira0435601-171 2014-07-01 16:41:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: done (15 left) 2014-07-01 16:41:49,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1766: done (14 left) 2014-07-01 16:41:49,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: done (13 left) 2014-07-01 16:41:49,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: done (12 left) 2014-07-01 16:41:49,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1773: done (11 left) 2014-07-01 16:41:49,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1776: done (10 left) 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1777: done (9 left) 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1778: done (8 left) 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 119 current snap is 0 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: seq_num 1195 ranges {1863680=536576,2400256=548864,2949120=655360} 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: writing mira0435601-119 from 1863680 to 2400256 tid 1 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: writing mira0435601-119 from 2400256 to 2949120 tid 2 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: writing mira0435601-119 from 2949120 to 3604480 tid 3 2014-07-01 16:41:50,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 278 current snap is 0 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: seq_num 1196 ranges {2154496=409600,2564096=536576,3100672=667648} 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0435601-278 from 2154496 to 2564096 tid 1 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0435601-278 from 2564096 to 3100672 tid 2 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0435601-278 from 3100672 to 3768320 tid 3 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: read oid 243 snap -1 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: seq_num 1197 ranges {4767744=438272} 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: writing mira0435601-400 from 4767744 to 5206016 tid 1 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: read oid 474 snap -1 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1786: read oid 228 snap -1 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 100 current snap is 0 2014-07-01 16:41:50,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: seq_num 1198 ranges {3641344=307200} 2014-07-01 16:41:50,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: writing mira0435601-100 from 3641344 to 3948544 tid 1 2014-07-01 16:41:50,109.109 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:50,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 60 current snap is 0 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: seq_num 1199 ranges {2232320=180224} 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: writing mira0435601-60 from 2232320 to 2412544 tid 1 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1786: done (15 left) 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1789: read oid 283 snap -1 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1789: done (15 left) 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1790: read oid 283 snap -1 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1790: done (15 left) 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 50 current snap is 0 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: seq_num 1200 ranges {2478080=655360,3133440=516096,3649536=774144,4423680=315392} 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: writing mira0435601-50 from 2478080 to 3133440 tid 1 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: writing mira0435601-50 from 3133440 to 3649536 tid 2 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: writing mira0435601-50 from 3649536 to 4423680 tid 3 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: writing mira0435601-50 from 4423680 to 4739072 tid 4 2014-07-01 16:41:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:50,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: finishing write tid 1 to mira0435601-400 2014-07-01 16:41:50,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: finishing write tid 3 to mira0435601-400 2014-07-01 16:41:50,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 1 to mira0435601-100 2014-07-01 16:41:50,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 3 to mira0435601-100 2014-07-01 16:41:50,552.552 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1544 16 6540 6524 16.8984 1.33333 3.34778 3.76698 2014-07-01 16:41:50,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: finishing write tid 4 to mira0435601-400 2014-07-01 16:41:50,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 207 (ObjNum 1197 snap 0 seq_num 1197) dirty exists 2014-07-01 16:41:50,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: done (15 left) 2014-07-01 16:41:50,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 145 current snap is 0 2014-07-01 16:41:50,673.673 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: seq_num 1201 ranges {1830912=528384,2359296=745472,3104768=602112,3706880=258048} 2014-07-01 16:41:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: writing mira0435601-145 from 1830912 to 2359296 tid 1 2014-07-01 16:41:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: writing mira0435601-145 from 2359296 to 3104768 tid 2 2014-07-01 16:41:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: writing mira0435601-145 from 3104768 to 3706880 tid 3 2014-07-01 16:41:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: writing mira0435601-145 from 3706880 to 3964928 tid 4 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 4 to mira0435601-100 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 209 (ObjNum 1198 snap 0 seq_num 1198) dirty exists 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: done (15 left) 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:41:50,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: seq_num 1202 ranges {6320128=802816,7122944=729088,7852032=475136} 2014-07-01 16:41:50,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0435601-439 from 6320128 to 7122944 tid 1 2014-07-01 16:41:50,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0435601-439 from 7122944 to 7852032 tid 2 2014-07-01 16:41:50,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0435601-439 from 7852032 to 8327168 tid 3 2014-07-01 16:41:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:51,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: finishing write tid 4 to mira0435601-40 2014-07-01 16:41:51,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 128 (ObjNum 1190 snap 0 seq_num 1190) dirty exists 2014-07-01 16:41:51,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: done (15 left) 2014-07-01 16:41:51,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 451 current snap is 0 2014-07-01 16:41:51,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: seq_num 1203 ranges {0=425984,425984=401408} 2014-07-01 16:41:51,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: writing mira0435601-451 from 0 to 425984 tid 1 2014-07-01 16:41:51,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: writing mira0435601-451 from 425984 to 827392 tid 2 2014-07-01 16:41:51,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:51,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: expect (ObjNum 876 snap 0 seq_num 876) 2014-07-01 16:41:51,625.625 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1545 16 6548 6532 16.9081 32 9.10314 3.77728 2014-07-01 16:41:52,435.435 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:52,552.552 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1546 16 6557 6541 16.9205 36 0.203976 3.77349 2014-07-01 16:41:53,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: finishing write tid 1 to mira0435601-198 2014-07-01 16:41:53,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: finishing write tid 3 to mira0435601-198 2014-07-01 16:41:53,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 1 to mira0435601-247 2014-07-01 16:41:53,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 2 to mira0435601-247 2014-07-01 16:41:53,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 3 to mira0435601-247 2014-07-01 16:41:53,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 4 to mira0435601-247 2014-07-01 16:41:53,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 6 to mira0435601-247 2014-07-01 16:41:53,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 1 to mira0435601-439 2014-07-01 16:41:53,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 2 to mira0435601-439 2014-07-01 16:41:53,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 3 to mira0435601-439 2014-07-01 16:41:53,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 5 to mira0435601-439 2014-07-01 16:41:53,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 6 to mira0435601-439 2014-07-01 16:41:53,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 213 (ObjNum 1202 snap 0 seq_num 1202) dirty exists 2014-07-01 16:41:53,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: done (15 left) 2014-07-01 16:41:53,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: done (14 left) 2014-07-01 16:41:53,149.149 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:40:54.535602', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886716228, u'health': u'HEALTH_OK', u'kb_used': 10064824, u'store_stats': {u'bytes_total': 6060952, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'49 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2608/14381 objects degraded (18.135%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2608, u'num_pgs': 244, u'data_bytes': 28386711740, u'degraded_total': 14381, u'bytes_used': 54470131712, u'op_per_sec': 2, u'write_bytes_sec': 370219, u'version': 1307, u'pgs_by_state': [{u'count': 26, u'state_name': u'active+recovery_wait'}, {u'count': 195, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1017, u'degraded_ratio': u'18.135', u'bytes_avail': 4944112406528}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 592, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:41:53,150.150 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: read oid 83 snap -1 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 433 current snap is 0 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: seq_num 1204 ranges {921600=749568,1671168=495616,2166784=307200} 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: writing mira0435601-433 from 921600 to 1671168 tid 1 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: writing mira0435601-433 from 1671168 to 2166784 tid 2 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: writing mira0435601-433 from 2166784 to 2473984 tid 3 2014-07-01 16:41:53,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:41:53,552.552 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1547 16 6557 6541 16.9096 0 - 3.77349 2014-07-01 16:41:53,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:1780: expect (ObjNum 953 snap 0 seq_num 953) 2014-07-01 16:41:53,929.929 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:41:53,929.929 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:54,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1548 16 6558 6542 16.9012 2 1.39304 3.77313 2014-07-01 16:41:54,748.748 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:41:55,553.553 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1549 16 6558 6542 16.8903 0 - 3.77313 2014-07-01 16:41:56,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: finishing write tid 4 to mira0435601-198 2014-07-01 16:41:56,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 172 (ObjNum 1189 snap 0 seq_num 1189) dirty exists 2014-07-01 16:41:56,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: finishing write tid 7 to mira0435601-459 2014-07-01 16:41:56,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 243 (ObjNum 1192 snap 0 seq_num 1192) dirty exists 2014-07-01 16:41:56,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: finishing write tid 7 to mira0435601-247 2014-07-01 16:41:56,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 217 (ObjNum 1193 snap 0 seq_num 1193) dirty exists 2014-07-01 16:41:56,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: finishing write tid 1 to mira0435601-119 2014-07-01 16:41:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: finishing write tid 2 to mira0435601-119 2014-07-01 16:41:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: finishing write tid 5 to mira0435601-192 2014-07-01 16:41:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 211 (ObjNum 1191 snap 0 seq_num 1191) dirty exists 2014-07-01 16:41:56,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: finishing write tid 3 to mira0435601-119 2014-07-01 16:41:56,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: finishing write tid 5 to mira0435601-171 2014-07-01 16:41:56,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 214 (ObjNum 1194 snap 0 seq_num 1194) dirty exists 2014-07-01 16:41:56,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 1 to mira0435601-50 2014-07-01 16:41:56,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: done (15 left) 2014-07-01 16:41:56,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: done (14 left) 2014-07-01 16:41:56,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: done (13 left) 2014-07-01 16:41:56,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: done (12 left) 2014-07-01 16:41:56,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: done (11 left) 2014-07-01 16:41:56,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1780: done (10 left) 2014-07-01 16:41:56,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: read oid 169 snap -1 2014-07-01 16:41:56,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 2 to mira0435601-50 2014-07-01 16:41:56,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 163 current snap is 0 2014-07-01 16:41:56,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: seq_num 1205 ranges {2199552=622592,2822144=569344,3391488=233472} 2014-07-01 16:41:56,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: writing mira0435601-163 from 2199552 to 2822144 tid 1 2014-07-01 16:41:56,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: writing mira0435601-163 from 2822144 to 3391488 tid 2 2014-07-01 16:41:56,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: writing mira0435601-163 from 3391488 to 3624960 tid 3 2014-07-01 16:41:56,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 363 current snap is 0 2014-07-01 16:41:56,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: seq_num 1206 ranges {0=245760} 2014-07-01 16:41:56,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: writing mira0435601-363 from 0 to 245760 tid 1 2014-07-01 16:41:56,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1800: delete oid 360 current snap is 0 2014-07-01 16:41:56,608.608 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1550 16 6572 6556 16.9155 28 0.584322 3.77827 2014-07-01 16:41:56,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 1 to mira0435601-163 2014-07-01 16:41:56,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 2 to mira0435601-163 2014-07-01 16:41:56,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 3 to mira0435601-163 2014-07-01 16:41:56,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 5 to mira0435601-163 2014-07-01 16:41:56,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 16:41:57,554.554 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1551 16 6576 6560 16.9149 16 0.67327 3.77641 2014-07-01 16:41:58,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 6 to mira0435601-163 2014-07-01 16:41:58,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 301 (ObjNum 1205 snap 0 seq_num 1205) dirty exists 2014-07-01 16:41:58,554.554 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1552 16 6578 6562 16.9092 8 1.53062 3.77572 2014-07-01 16:41:59,136.136 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:41:59,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1553 16 6578 6562 16.8983 0 - 3.77572 2014-07-01 16:41:59,894.894 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:00,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1554 16 6578 6562 16.8874 0 - 3.77572 2014-07-01 16:42:01,445.445 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:01,554.554 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1555 16 6578 6562 16.8766 0 - 3.77572 2014-07-01 16:42:02,186.186 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:41:11.318528', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886710428, u'health': u'HEALTH_OK', u'kb_used': 10070624, u'store_stats': {u'bytes_total': 7109549, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963805}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'24 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'17 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2578/14454 objects degraded (17.836%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 7940874, u'degraded_objects': 2578, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28546259110, u'degraded_total': 14454, u'bytes_used': 54430117888, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 1323479, u'op_per_sec': 0, u'version': 1314, u'pgs_by_state': [{u'count': 24, u'state_name': u'active+recovery_wait'}, {u'count': 198, u'state_name': u'active+clean'}, {u'count': 22, u'state_name': u'active+recovering'}], u'read_bytes_sec': 327315, u'degraded_ratio': u'17.836', u'bytes_avail': 4944152420352}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 594, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:02,187.187 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:02,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 3 to mira0435601-50 2014-07-01 16:42:02,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: finishing write tid 5 to mira0435601-119 2014-07-01 16:42:02,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 1 to mira0435601-278 2014-07-01 16:42:02,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 2 to mira0435601-278 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 3 to mira0435601-278 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 4 to mira0435601-50 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 6 to mira0435601-50 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 5 to mira0435601-278 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 1 to mira0435601-60 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 3 to mira0435601-60 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: finishing write tid 1 to mira0435601-451 2014-07-01 16:42:02,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: finishing write tid 2 to mira0435601-451 2014-07-01 16:42:02,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: finishing write tid 4 to mira0435601-451 2014-07-01 16:42:02,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 4 to mira0435601-60 2014-07-01 16:42:02,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 60 v 107 (ObjNum 1199 snap 0 seq_num 1199) dirty exists 2014-07-01 16:42:02,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: expect (ObjNum 905 snap 0 seq_num 905) 2014-07-01 16:42:02,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1556 16 6588 6572 16.8914 10 0.367297 3.78091 2014-07-01 16:42:02,943.943 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:42:02,944.944 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:03,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1557 16 6592 6576 16.8909 16 0.097567 3.77877 2014-07-01 16:42:03,688.688 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:04,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 1 to mira0435601-145 2014-07-01 16:42:04,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: expect (ObjNum 82 snap 0 seq_num 82) 2014-07-01 16:42:04,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1558 16 6592 6576 16.88 0 - 3.77877 2014-07-01 16:42:05,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1559 16 6592 6576 16.8692 0 - 3.77877 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 2 to mira0435601-145 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 3 to mira0435601-145 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 4 to mira0435601-145 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 6 to mira0435601-145 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: finishing write tid 1 to mira0435601-433 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: finishing write tid 2 to mira0435601-433 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: finishing write tid 3 to mira0435601-433 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: finishing write tid 5 to mira0435601-433 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: finishing write tid 1 to mira0435601-363 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: finishing write tid 3 to mira0435601-363 2014-07-01 16:42:05,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: done (13 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: done (12 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: done (11 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: done (10 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: done (9 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1800: done (8 left) 2014-07-01 16:42:05,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 16:42:05,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: seq_num 1208 ranges {315392=794624,1110016=720896,1830912=667648,2498560=217088} 2014-07-01 16:42:05,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: writing mira0435601-397 from 315392 to 1110016 tid 1 2014-07-01 16:42:05,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: writing mira0435601-397 from 1110016 to 1830912 tid 2 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: writing mira0435601-397 from 1830912 to 2498560 tid 3 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: writing mira0435601-397 from 2498560 to 2715648 tid 4 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 73 current snap is 0 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: seq_num 1209 ranges {1638400=466944,2105344=552960,2658304=741376,3399680=311296} 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: writing mira0435601-73 from 1638400 to 2105344 tid 1 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: writing mira0435601-73 from 2105344 to 2658304 tid 2 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: writing mira0435601-73 from 2658304 to 3399680 tid 3 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: writing mira0435601-73 from 3399680 to 3710976 tid 4 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: read oid 69 snap -1 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 466 current snap is 0 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: seq_num 1210 ranges {0=495616,495616=401408} 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: writing mira0435601-466 from 0 to 495616 tid 1 2014-07-01 16:42:05,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: writing mira0435601-466 from 495616 to 897024 tid 2 2014-07-01 16:42:06,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1805: read oid 87 snap -1 2014-07-01 16:42:06,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1806: delete oid 331 current snap is 0 2014-07-01 16:42:06,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1806: done (13 left) 2014-07-01 16:42:06,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: read oid 141 snap -1 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1808: read oid 254 snap -1 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 456 current snap is 0 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: seq_num 1212 ranges {1064960=638976} 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: writing mira0435601-456 from 1064960 to 1703936 tid 1 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:42:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: expect (ObjNum 1027 snap 0 seq_num 1027) 2014-07-01 16:42:06,459.459 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_set_snap_recreated ... ok 2014-07-01 16:42:06,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:42:06.467236min lat: 0.057269 max lat: 294.525 avg lat: 3.78588 2014-07-01 16:42:06,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:42:06,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1560 16 6603 6587 16.8866 14.6667 0.211547 3.78588 2014-07-01 16:42:07,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1561 16 6606 6590 16.8834 12 0.447367 3.78433 2014-07-01 16:42:07,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: finishing write tid 7 to mira0435601-50 2014-07-01 16:42:07,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 166 (ObjNum 1200 snap 0 seq_num 1200) dirty exists 2014-07-01 16:42:07,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 6 to mira0435601-278 2014-07-01 16:42:07,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 243 (ObjNum 1196 snap 0 seq_num 1196) dirty exists 2014-07-01 16:42:07,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: expect (ObjNum 668 snap 0 seq_num 668) 2014-07-01 16:42:08,222.222 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:08,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1562 16 6616 6600 16.8982 40 2.16139 3.78541 2014-07-01 16:42:08,818.818 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:09,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1563 16 6629 6613 16.9207 52 0.942137 3.78074 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: finishing write tid 6 to mira0435601-433 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 195 (ObjNum 1204 snap 0 seq_num 1204) dirty exists 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: finishing write tid 5 to mira0435601-451 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 211 (ObjNum 1203 snap 0 seq_num 1203) dirty exists 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: finishing write tid 4 to mira0435601-363 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 216 (ObjNum 1206 snap 0 seq_num 1206) dirty exists 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: finishing write tid 7 to mira0435601-145 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 99 (ObjNum 1201 snap 0 seq_num 1201) dirty exists 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 1 to mira0435601-73 2014-07-01 16:42:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: done (15 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: done (14 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: done (13 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: done (12 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: done (11 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: done (10 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: done (9 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: done (8 left) 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: read oid 374 snap -1 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: finishing write tid 6 to mira0435601-119 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 208 (ObjNum 1195 snap 0 seq_num 1195) dirty exists 2014-07-01 16:42:09,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:1808: expect (ObjNum 253 snap 0 seq_num 253) 2014-07-01 16:42:10,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 1 to mira0435601-397 2014-07-01 16:42:10,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 2 to mira0435601-397 2014-07-01 16:42:10,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 2 to mira0435601-73 2014-07-01 16:42:10,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 3 to mira0435601-73 2014-07-01 16:42:10,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 4 to mira0435601-73 2014-07-01 16:42:10,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1805: expect (ObjNum 86 snap 0 seq_num 86) 2014-07-01 16:42:10,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: expect (ObjNum 68 snap 0 seq_num 68) 2014-07-01 16:42:10,568.568 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:10,569.569 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1564 16 6629 6613 16.9099 0 - 3.78074 2014-07-01 16:42:11,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 6 to mira0435601-73 2014-07-01 16:42:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 3 to mira0435601-397 2014-07-01 16:42:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 4 to mira0435601-397 2014-07-01 16:42:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 6 to mira0435601-397 2014-07-01 16:42:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: finishing write tid 1 to mira0435601-466 2014-07-01 16:42:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: done (8 left) 2014-07-01 16:42:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: done (7 left) 2014-07-01 16:42:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1805: done (6 left) 2014-07-01 16:42:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:1808: done (5 left) 2014-07-01 16:42:11,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 16:42:11,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: seq_num 1213 ranges {786432=315392} 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: writing mira0435601-186 from 786432 to 1101824 tid 1 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: read oid 38 snap -1 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: finishing write tid 2 to mira0435601-466 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: read oid 262 snap -1 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: finishing write tid 4 to mira0435601-466 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 1 to mira0435601-456 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 3 to mira0435601-456 2014-07-01 16:42:11,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1814: delete oid 274 current snap is 0 2014-07-01 16:42:11,555.555 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1565 16 6630 6614 16.9016 2 1.96521 3.78046 2014-07-01 16:42:12,557.557 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1566 16 6630 6614 16.8908 0 - 3.78046 2014-07-01 16:42:13,033.033 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2571/14533 objects degraded (17.691%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12514775, u'degraded_objects': 2571, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28713099185, u'degraded_total': 14533, u'bytes_used': 54988664832, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 12292242, u'op_per_sec': 7, u'version': 1323, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 198, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 37682, u'degraded_ratio': u'17.691', u'bytes_avail': 4943593873408}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 595, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:13,034.034 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:13,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1567 16 6630 6614 16.88 0 - 3.78046 2014-07-01 16:42:13,741.741 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:42:13,741.741 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:14,557.557 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1568 16 6632 6616 16.8744 2.66667 4.43144 3.78065 2014-07-01 16:42:14,705.705 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:15,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 7 to mira0435601-397 2014-07-01 16:42:15,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 125 (ObjNum 1208 snap 0 seq_num 1208) dirty exists 2014-07-01 16:42:15,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: finishing write tid 5 to mira0435601-466 2014-07-01 16:42:15,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 169 (ObjNum 1210 snap 0 seq_num 1210) dirty exists 2014-07-01 16:42:15,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: expect (ObjNum 1164 snap 0 seq_num 1164) 2014-07-01 16:42:15,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1569 16 6633 6617 16.8662 4 4.08643 3.78069 2014-07-01 16:42:16,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1570 16 6635 6619 16.8605 8 2.42612 3.78099 2014-07-01 16:42:17,345.345 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_set_snap_sparse ... ok 2014-07-01 16:42:17,557.557 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1571 16 6635 6619 16.8498 0 - 3.78099 2014-07-01 16:42:18,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 4 to mira0435601-456 2014-07-01 16:42:18,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 174 (ObjNum 1212 snap 0 seq_num 1212) dirty exists 2014-07-01 16:42:18,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: finishing write tid 7 to mira0435601-73 2014-07-01 16:42:18,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 182 (ObjNum 1209 snap 0 seq_num 1209) dirty exists 2014-07-01 16:42:18,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: expect (ObjNum 1159 snap 0 seq_num 1159) 2014-07-01 16:42:18,557.557 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1572 16 6646 6630 16.8671 22 0.214419 3.78355 2014-07-01 16:42:19,275.275 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:19,556.556 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1573 16 6654 6638 16.8767 32 0.208066 3.78393 2014-07-01 16:42:20,049.049 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:20,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1574 16 6654 6638 16.866 0 - 3.78393 2014-07-01 16:42:21,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: expect (ObjNum 1094 snap 0 seq_num 1094) 2014-07-01 16:42:21,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1575 16 6654 6638 16.8552 0 - 3.78393 2014-07-01 16:42:21,590.590 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:22,332.332 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2585/14603 objects degraded (17.702%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5550368, u'degraded_objects': 2585, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28861591451, u'degraded_total': 14603, u'bytes_used': 55378784256, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 15264191, u'op_per_sec': 8, u'version': 1330, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 198, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'read_bytes_sec': 516620, u'degraded_ratio': u'17.702', u'bytes_avail': 4943203753984}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 597, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:22,333.333 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:22,560.560 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1576 16 6657 6641 16.8522 4 0.415474 3.78316 2014-07-01 16:42:23,115.115 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:42:23,116.116 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:23,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1577 16 6657 6641 16.8415 0 - 3.78316 2014-07-01 16:42:23,914.914 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:24,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1578 16 6662 6646 16.8435 10 14.9857 3.79002 2014-07-01 16:42:25,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1579 16 6662 6646 16.8328 0 - 3.79002 2014-07-01 16:42:26,560.560 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:42:26.470694min lat: 0.057269 max lat: 294.525 avg lat: 3.79088 2014-07-01 16:42:26,560.560 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:42:26,561.561 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1580 16 6684 6668 16.8778 44 0.250919 3.79088 2014-07-01 16:42:27,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1581 16 6691 6675 16.8849 28 0.154707 3.78723 2014-07-01 16:42:28,147.147 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_size ... ok 2014-07-01 16:42:28,474.474 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:28,558.558 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1582 16 6692 6676 16.8767 4 3.41433 3.78717 2014-07-01 16:42:29,063.063 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:29,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1583 16 6692 6676 16.8661 0 - 3.78717 2014-07-01 16:42:30,560.560 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1584 16 6692 6676 16.8554 0 - 3.78717 2014-07-01 16:42:30,814.814 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:31,519.519 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'46 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'8 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2598/14689 objects degraded (17.687%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2598, u'num_pgs': 244, u'data_bytes': 29041946545, u'degraded_total': 14689, u'bytes_used': 55804755968, u'op_per_sec': 0, u'write_bytes_sec': 3835949, u'version': 1340, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 198, u'state_name': u'active+clean'}, {u'count': 23, u'state_name': u'active+recovering'}], u'degraded_ratio': u'17.687', u'bytes_avail': 4942777782272}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:31,520.520 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:31,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1585 16 6692 6676 16.8448 0 - 3.78717 2014-07-01 16:42:32,298.298 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:42:32,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:32,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1586 16 6692 6676 16.8342 0 - 3.78717 2014-07-01 16:42:33,101.101 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:33,559.559 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1587 16 6704 6688 16.8538 9.6 0.191712 3.78754 2014-07-01 16:42:34,569.569 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1588 16 6705 6689 16.8457 4 0.191711 3.78701 2014-07-01 16:42:35,562.562 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1589 16 6705 6689 16.8351 0 - 3.78701 2014-07-01 16:42:35,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: finishing write tid 1 to mira0435601-186 2014-07-01 16:42:35,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: finishing write tid 3 to mira0435601-186 2014-07-01 16:42:36,561.561 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1590 16 6716 6700 16.8522 22 0.850089 3.78806 2014-07-01 16:42:37,561.561 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1591 16 6716 6700 16.8416 0 - 3.78806 2014-07-01 16:42:37,664.664 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:38,430.430 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:38,561.561 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1592 16 6717 6701 16.8335 2 2.19305 3.78782 2014-07-01 16:42:39,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: done (8 left) 2014-07-01 16:42:39,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: done (7 left) 2014-07-01 16:42:39,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: done (6 left) 2014-07-01 16:42:39,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: done (5 left) 2014-07-01 16:42:39,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: done (4 left) 2014-07-01 16:42:39,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: done (3 left) 2014-07-01 16:42:39,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: done (2 left) 2014-07-01 16:42:39,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1814: done (1 left) 2014-07-01 16:42:39,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:1815: delete oid 318 current snap is 0 2014-07-01 16:42:39,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: finishing write tid 4 to mira0435601-186 2014-07-01 16:42:39,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 119 (ObjNum 1213 snap 0 seq_num 1213) dirty exists 2014-07-01 16:42:39,504.504 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_stat ... ok 2014-07-01 16:42:39,653.653 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1593 16 6730 6714 16.8556 52 0.310488 3.79419 2014-07-01 16:42:39,975.975 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:40,561.561 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1594 16 6740 6724 16.8701 40 0.171532 3.78886 2014-07-01 16:42:40,705.705 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:41:41.905145', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883109512, u'health': u'HEALTH_OK', u'kb_used': 13671540, u'store_stats': {u'bytes_total': 7110772, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3965028}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'44 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2565/14739 objects degraded (17.403%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2565, u'num_pgs': 244, u'data_bytes': 29147119526, u'degraded_total': 14739, u'bytes_used': 56226312192, u'op_per_sec': 14, u'write_bytes_sec': 36268379, u'version': 1345, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 200, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2449, u'degraded_ratio': u'17.403', u'bytes_avail': 4942356226048}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:40,705.705 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:41,562.562 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1595 16 6740 6724 16.8595 0 - 3.78886 2014-07-01 16:42:42,563.563 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1596 16 6740 6724 16.849 0 - 3.78886 2014-07-01 16:42:43,563.563 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1597 16 6740 6724 16.8384 0 - 3.78886 2014-07-01 16:42:44,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1598 16 6740 6724 16.8279 0 - 3.78886 2014-07-01 16:42:44,937.937 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:42:44,937.937 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:45,562.562 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1599 16 6740 6724 16.8174 0 - 3.78886 2014-07-01 16:42:45,815.815 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:46,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:42:46.474934min lat: 0.057269 max lat: 294.525 avg lat: 3.78886 2014-07-01 16:42:46,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:42:46,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1600 16 6740 6724 16.8068 0 - 3.78886 2014-07-01 16:42:47,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1601 16 6754 6738 16.8313 8 0.131719 3.79452 2014-07-01 16:42:48,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1602 16 6754 6738 16.8208 0 - 3.79452 2014-07-01 16:42:49,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1603 16 6774 6758 16.8602 40 0.123377 3.79372 2014-07-01 16:42:50,370.370 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:50,563.563 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1604 16 6778 6762 16.8597 16 0.117244 3.7917 2014-07-01 16:42:51,146.146 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:51,563.563 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1605 16 6778 6762 16.8492 0 - 3.7917 2014-07-01 16:42:52,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1606 16 6778 6762 16.8387 0 - 3.7917 2014-07-01 16:42:52,713.713 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:53,429.429 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:41:54.535922', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886693736, u'health': u'HEALTH_OK', u'kb_used': 10087316, u'store_stats': {u'bytes_total': 9206680, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3963784}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'44 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2588/14858 objects degraded (17.418%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 2624703, u'degraded_objects': 2588, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29396168870, u'degraded_total': 14858, u'bytes_used': 56386174976, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 50081801, u'op_per_sec': 15, u'version': 1352, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 200, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2037, u'degraded_ratio': u'17.418', u'bytes_avail': 4942196363264}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:42:53,429.429 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:42:53,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1607 16 6786 6770 16.8481 10.6667 0.364078 3.79 2014-07-01 16:42:54,206.206 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:42:54,206.206 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:54,546.546 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_write ... ok 2014-07-01 16:42:54,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1608 16 6786 6770 16.8376 0 - 3.79 2014-07-01 16:42:55,015.015 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:42:55,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1609 16 6786 6770 16.8272 0 - 3.79 2014-07-01 16:42:56,564.564 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1610 16 6786 6770 16.8167 0 - 3.79 2014-07-01 16:42:57,565.565 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1611 16 6786 6770 16.8063 0 - 3.79 2014-07-01 16:42:58,566.566 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1612 16 6786 6770 16.7958 0 - 3.79 2014-07-01 16:42:59,543.543 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:42:59,570.570 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1613 16 6798 6782 16.8152 8 6.27183 3.80221 2014-07-01 16:43:00,309.309 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:00,568.568 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1614 16 6812 6796 16.8395 56 0.64862 3.79919 2014-07-01 16:43:01,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1615 16 6816 6800 16.8389 16 0.116285 3.79707 2014-07-01 16:43:01,869.869 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:02,566.566 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1616 16 6816 6800 16.8285 0 - 3.79707 2014-07-01 16:43:02,572.572 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:42:11.318846', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886696100, u'health': u'HEALTH_OK', u'kb_used': 10084952, u'store_stats': {u'bytes_total': 6066212, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3969044}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2580/14888 objects degraded (17.329%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 7345860, u'degraded_objects': 2580, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29459083469, u'degraded_total': 14888, u'bytes_used': 56618741760, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 7346373, u'op_per_sec': 4, u'version': 1357, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 201, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1281, u'degraded_ratio': u'17.329', u'bytes_avail': 4941963796480}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:02,573.573 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:03,318.318 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:03,319.319 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:03,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1617 16 6819 6803 16.8255 6 3.03396 3.79702 2014-07-01 16:43:04,144.144 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:04,566.566 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1618 16 6819 6803 16.8151 0 - 3.79702 2014-07-01 16:43:05,383.383 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.TestImage.test_write_read ... ok 2014-07-01 16:43:05,396.396 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_version ... ok 2014-07-01 16:43:05,614.614 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1619 16 6824 6808 16.8171 10 1.85716 3.79758 2014-07-01 16:43:06,642.642 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:43:06.479070min lat: 0.057269 max lat: 294.525 avg lat: 3.79545 2014-07-01 16:43:06,642.642 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:43:06,643.643 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1620 16 6828 6812 16.8166 16 0.150037 3.79545 2014-07-01 16:43:07,566.566 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1621 16 6828 6812 16.8062 0 - 3.79545 2014-07-01 16:43:08,568.568 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1622 16 6828 6812 16.7959 0 - 3.79545 2014-07-01 16:43:08,711.711 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:09,510.510 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:09,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1623 16 6828 6812 16.7855 0 - 3.79545 2014-07-01 16:43:10,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1624 16 6828 6812 16.7752 0 - 3.79545 2014-07-01 16:43:11,234.234 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:11,380.380 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_create ... ok 2014-07-01 16:43:11,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1625 16 6828 6812 16.7648 0 - 3.79545 2014-07-01 16:43:11,941.941 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'10 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2587/14954 objects degraded (17.300%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 14223964, u'degraded_objects': 2587, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29605883691, u'degraded_total': 14954, u'bytes_used': 57027203072, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 16002394, u'op_per_sec': 5, u'version': 1361, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 201, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 434, u'degraded_ratio': u'17.300', u'bytes_avail': 4941555335168}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:11,941.941 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:12,567.567 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1626 16 6828 6812 16.7545 0 - 3.79545 2014-07-01 16:43:12,706.706 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:12,707.707 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:13,500.500 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:13,568.568 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1627 16 6828 6812 16.7442 0 - 3.79545 2014-07-01 16:43:14,568.568 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1628 16 6828 6812 16.7339 0 - 3.79545 2014-07-01 16:43:15,570.570 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1629 16 6828 6812 16.7237 0 - 3.79545 2014-07-01 16:43:16,569.569 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1630 16 6828 6812 16.7134 0 - 3.79545 2014-07-01 16:43:17,570.570 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1631 16 6828 6812 16.7032 0 - 3.79545 2014-07-01 16:43:17,891.891 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:18,569.569 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1632 16 6828 6812 16.6929 0 - 3.79545 2014-07-01 16:43:18,665.665 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:19,571.571 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1633 16 6828 6812 16.6827 0 - 3.79545 2014-07-01 16:43:20,216.216 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:20,569.569 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1634 16 6828 6812 16.6725 0 - 3.79545 2014-07-01 16:43:20,953.953 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'10 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2575/14960 objects degraded (17.213%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 11161871, u'degraded_objects': 2575, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29614272422, u'degraded_total': 14960, u'bytes_used': 57248866304, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 1860993, u'op_per_sec': 1, u'version': 1364, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 201, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 681, u'degraded_ratio': u'17.213', u'bytes_avail': 4941333671936}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:20,953.953 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:21,570.570 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1635 16 6828 6812 16.6623 0 - 3.79545 2014-07-01 16:43:21,725.725 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:21,725.725 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:22,543.543 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:22,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1636 16 6828 6812 16.6521 0 - 3.79545 2014-07-01 16:43:23,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1637 16 6828 6812 16.6419 0 - 3.79545 2014-07-01 16:43:24,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1638 16 6828 6812 16.6318 0 - 3.79545 2014-07-01 16:43:25,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1639 16 6828 6812 16.6216 0 - 3.79545 2014-07-01 16:43:26,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:43:26.483518min lat: 0.057269 max lat: 294.525 avg lat: 3.79545 2014-07-01 16:43:26,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:43:26,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1640 16 6828 6812 16.6115 0 - 3.79545 2014-07-01 16:43:27,064.064 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:27,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1641 16 6828 6812 16.6014 0 - 3.79545 2014-07-01 16:43:27,831.831 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:28,571.571 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1642 16 6828 6812 16.5913 0 - 3.79545 2014-07-01 16:43:29,390.390 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:29,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1643 16 6828 6812 16.5812 0 - 3.79545 2014-07-01 16:43:30,101.101 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'21 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2563/14964 objects degraded (17.128%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5258392, u'degraded_objects': 2563, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29626855211, u'degraded_total': 14964, u'bytes_used': 56247029760, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 1051764, u'op_per_sec': 0, u'version': 1366, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 201, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 85, u'degraded_ratio': u'17.128', u'bytes_avail': 4942335508480}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:30,102.102 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:30,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1644 16 6842 6826 16.6051 2.33333 0.137834 3.84253 2014-07-01 16:43:30,900.900 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:30,900.900 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:31,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1645 16 6859 6843 16.6364 68 1.06284 3.8455 2014-07-01 16:43:32,106.106 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:32,572.572 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1646 16 6861 6845 16.6311 8 1.44593 3.8448 2014-07-01 16:43:33,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1647 16 6869 6853 16.6405 32 1.78898 3.84252 2014-07-01 16:43:34,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: done (1 left) 2014-07-01 16:43:34,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1815: done (0 left) 2014-07-01 16:43:34,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 316 current snap is 0 2014-07-01 16:43:34,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: seq_num 1216 ranges {659456=487424,1146880=528384,1675264=274432} 2014-07-01 16:43:34,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: writing mira0435601-316 from 659456 to 1146880 tid 1 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: writing mira0435601-316 from 1146880 to 1675264 tid 2 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: writing mira0435601-316 from 1675264 to 1949696 tid 3 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: delete oid 239 current snap is 0 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: done (1 left) 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1818: read oid 335 snap -1 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1819: read oid 255 snap -1 2014-07-01 16:43:34,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1820: delete oid 371 current snap is 0 2014-07-01 16:43:34,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1648 16 6878 6862 16.6522 36 0.6326 3.83922 2014-07-01 16:43:34,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: finishing write tid 1 to mira0435601-316 2014-07-01 16:43:34,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: finishing write tid 2 to mira0435601-316 2014-07-01 16:43:34,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: finishing write tid 3 to mira0435601-316 2014-07-01 16:43:34,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: finishing write tid 5 to mira0435601-316 2014-07-01 16:43:35,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1649 16 6879 6863 16.6445 4 0.594669 3.83874 2014-07-01 16:43:36,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1650 16 6879 6863 16.6344 0 - 3.83874 2014-07-01 16:43:36,653.653 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:37,418.418 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:37,573.573 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1651 16 6879 6863 16.6244 0 - 3.83874 2014-07-01 16:43:38,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1652 16 6879 6863 16.6143 0 - 3.83874 2014-07-01 16:43:38,977.977 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:39,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1653 16 6879 6863 16.6042 0 - 3.83874 2014-07-01 16:43:39,680.680 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:42:44.711028', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883068824, u'health': u'HEALTH_OK', u'kb_used': 13712228, u'store_stats': {u'bytes_total': 7116054, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3970310}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2505/15045 objects degraded (16.650%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3849238, u'degraded_objects': 2505, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29788860219, u'degraded_total': 15045, u'bytes_used': 57100689408, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 6736168, u'op_per_sec': 11, u'version': 1371, u'pgs_by_state': [{u'count': 23, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 19, u'state_name': u'active+recovering'}], u'read_bytes_sec': 7283, u'degraded_ratio': u'16.650', u'bytes_avail': 4941481848832}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:39,680.680 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:40,488.488 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:40,489.489 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:40,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1654 16 6880 6864 16.5966 0.8 9.34771 3.83955 2014-07-01 16:43:41,284.284 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:41,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1655 16 6880 6864 16.5866 0 - 3.83955 2014-07-01 16:43:42,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1820: done (3 left) 2014-07-01 16:43:42,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1821: read oid 227 snap -1 2014-07-01 16:43:42,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: read oid 214 snap -1 2014-07-01 16:43:42,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: seq_num 1219 ranges {3948544=708608} 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: writing mira0435601-9 from 3948544 to 4657152 tid 1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1824: read oid 131 snap -1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1825: read oid 216 snap -1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: seq_num 1220 ranges {3514368=577536} 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: writing mira0435601-311 from 3514368 to 4091904 tid 1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1827: read oid 267 snap -1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 467 current snap is 0 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: seq_num 1221 ranges {0=450560} 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: writing mira0435601-467 from 0 to 450560 tid 1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 475 current snap is 0 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: seq_num 1222 ranges {3002368=126976} 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: writing mira0435601-475 from 3002368 to 3129344 tid 1 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 16:43:42,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: seq_num 1223 ranges {2523136=798720,3321856=503808,3825664=782336,4608000=319488} 2014-07-01 16:43:42,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: writing mira0435601-351 from 2523136 to 3321856 tid 1 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: writing mira0435601-351 from 3321856 to 3825664 tid 2 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: writing mira0435601-351 from 3825664 to 4608000 tid 3 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: writing mira0435601-351 from 4608000 to 4927488 tid 4 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1831: delete oid 448 current snap is 0 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: finishing write tid 1 to mira0435601-9 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: finishing write tid 3 to mira0435601-9 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: finishing write tid 4 to mira0435601-9 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 304 (ObjNum 1219 snap 0 seq_num 1219) dirty exists 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 1 to mira0435601-351 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 2 to mira0435601-351 2014-07-01 16:43:42,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 3 to mira0435601-351 2014-07-01 16:43:42,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 4 to mira0435601-351 2014-07-01 16:43:42,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 6 to mira0435601-351 2014-07-01 16:43:42,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: finishing write tid 7 to mira0435601-351 2014-07-01 16:43:42,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 309 (ObjNum 1223 snap 0 seq_num 1223) dirty exists 2014-07-01 16:43:42,643.643 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1656 16 6890 6874 16.6007 20 0.175713 3.84505 2014-07-01 16:43:42,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1824: expect (ObjNum 815 snap 0 seq_num 815) 2014-07-01 16:43:43,574.574 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1657 16 6903 6887 16.6221 52 0.480723 3.84881 2014-07-01 16:43:44,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1825: expect (ObjNum 215 snap 0 seq_num 215) 2014-07-01 16:43:44,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1658 16 6907 6891 16.6217 16 2.14888 3.84762 2014-07-01 16:43:44,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 1 to mira0435601-311 2014-07-01 16:43:44,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:1821: expect (ObjNum 572 snap 0 seq_num 572) 2014-07-01 16:43:45,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1659 16 6907 6891 16.6117 0 - 3.84762 2014-07-01 16:43:45,830.830 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:45,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1819: expect (ObjNum 254 snap 0 seq_num 254) 2014-07-01 16:43:46,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:43:46.487676min lat: 0.057269 max lat: 294.525 avg lat: 3.8475 2014-07-01 16:43:46,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:43:46,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1660 16 6908 6892 16.6041 2 3.03106 3.8475 2014-07-01 16:43:46,590.590 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:46,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: finishing write tid 6 to mira0435601-316 2014-07-01 16:43:46,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 170 (ObjNum 1216 snap 0 seq_num 1216) dirty exists 2014-07-01 16:43:46,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 3 to mira0435601-311 2014-07-01 16:43:46,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1818: expect (ObjNum 716 snap 0 seq_num 716) 2014-07-01 16:43:47,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1661 16 6908 6892 16.5941 0 - 3.8475 2014-07-01 16:43:48,155.155 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:48,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1662 16 6908 6892 16.5841 0 - 3.8475 2014-07-01 16:43:49,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1663 16 6908 6892 16.5741 0 - 3.8475 2014-07-01 16:43:50,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: expect (ObjNum 1121 snap 0 seq_num 1121) 2014-07-01 16:43:50,575.575 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1664 16 6908 6892 16.5642 0 - 3.8475 2014-07-01 16:43:51,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1665 16 6908 6892 16.5542 0 - 3.8475 2014-07-01 16:43:52,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1666 16 6908 6892 16.5443 0 - 3.8475 2014-07-01 16:43:52,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1827: expect (ObjNum 769 snap 0 seq_num 769) 2014-07-01 16:43:53,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1667 16 6908 6892 16.5344 0 - 3.8475 2014-07-01 16:43:53,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: finishing write tid 1 to mira0435601-467 2014-07-01 16:43:53,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: finishing write tid 3 to mira0435601-467 2014-07-01 16:43:53,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: done (13 left) 2014-07-01 16:43:53,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1818: done (12 left) 2014-07-01 16:43:53,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:1819: done (11 left) 2014-07-01 16:43:53,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:1821: done (10 left) 2014-07-01 16:43:53,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: done (9 left) 2014-07-01 16:43:54,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: done (8 left) 2014-07-01 16:43:54,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1824: done (7 left) 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1825: done (6 left) 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1827: done (5 left) 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: done (4 left) 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1831: done (3 left) 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1832: read oid 46 snap -1 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 461 current snap is 0 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: seq_num 1225 ranges {28672=544768,573440=720896,1294336=729088,2023424=155648} 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: writing mira0435601-461 from 28672 to 573440 tid 1 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: writing mira0435601-461 from 573440 to 1294336 tid 2 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: writing mira0435601-461 from 1294336 to 2023424 tid 3 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: writing mira0435601-461 from 2023424 to 2179072 tid 4 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: seq_num 1226 ranges {5206016=569344,5775360=401408} 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: writing mira0435601-400 from 5206016 to 5775360 tid 1 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: writing mira0435601-400 from 5775360 to 6176768 tid 2 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 235 current snap is 0 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: seq_num 1227 ranges {1978368=491520,2469888=729088,3198976=475136} 2014-07-01 16:43:54,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0435601-235 from 1978368 to 2469888 tid 1 2014-07-01 16:43:54,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0435601-235 from 2469888 to 3198976 tid 2 2014-07-01 16:43:54,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0435601-235 from 3198976 to 3674112 tid 3 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: seq_num 1228 ranges {3969024=765952,4734976=716800,5451776=86016} 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: writing mira0435601-359 from 3969024 to 4734976 tid 1 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: writing mira0435601-359 from 4734976 to 5451776 tid 2 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: writing mira0435601-359 from 5451776 to 5537792 tid 3 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 222 current snap is 0 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: seq_num 1229 ranges {2711552=389120} 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: writing mira0435601-222 from 2711552 to 3100672 tid 1 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 496 current snap is 0 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: seq_num 1230 ranges {3383296=417792,3801088=528384,4329472=446464,4775936=229376} 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0435601-496 from 3383296 to 3801088 tid 1 2014-07-01 16:43:54,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0435601-496 from 3801088 to 4329472 tid 2 2014-07-01 16:43:54,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0435601-496 from 4329472 to 4775936 tid 3 2014-07-01 16:43:54,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0435601-496 from 4775936 to 5005312 tid 4 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 449 current snap is 0 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: seq_num 1231 ranges {843776=585728,1429504=401408} 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: writing mira0435601-449 from 843776 to 1429504 tid 1 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: writing mira0435601-449 from 1429504 to 1830912 tid 2 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 458 current snap is 0 2014-07-01 16:43:54,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: seq_num 1232 ranges {2482176=610304,3092480=696320,3788800=499712,4288512=409600} 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: writing mira0435601-458 from 2482176 to 3092480 tid 1 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: writing mira0435601-458 from 3092480 to 3788800 tid 2 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: writing mira0435601-458 from 3788800 to 4288512 tid 3 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: writing mira0435601-458 from 4288512 to 4698112 tid 4 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: read oid 326 snap -1 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1842: read oid 185 snap -1 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: read oid 385 snap -1 2014-07-01 16:43:54,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1844: delete oid 307 current snap is 0 2014-07-01 16:43:54,573.573 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:42:54.536295', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886690328, u'health': u'HEALTH_OK', u'kb_used': 10090724, u'store_stats': {u'bytes_total': 4158409, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2502/15106 objects degraded (16.563%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 7299327, u'degraded_objects': 2502, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29934870315, u'degraded_total': 15106, u'bytes_used': 57911115776, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 14598654, u'op_per_sec': 8, u'version': 1375, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 4062338, u'degraded_ratio': u'16.563', u'bytes_avail': 4940671422464}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:43:54,574.574 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:43:54,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1668 16 6908 6892 16.5245 0 - 3.8475 2014-07-01 16:43:55,280.280 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:43:55,280.280 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:43:55,576.576 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1669 16 6908 6892 16.5146 0 - 3.8475 2014-07-01 16:43:56,233.233 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:43:56,596.596 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1670 16 6910 6894 16.5095 0.8 12.3919 3.84999 2014-07-01 16:43:57,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1671 16 6913 6897 16.5068 12 1.53937 3.85193 2014-07-01 16:43:58,578.578 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1672 16 6913 6897 16.4969 0 - 3.85193 2014-07-01 16:43:59,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 4 to mira0435601-311 2014-07-01 16:43:59,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 202 (ObjNum 1220 snap 0 seq_num 1220) dirty exists 2014-07-01 16:43:59,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 1 to mira0435601-400 2014-07-01 16:43:59,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1673 16 6916 6900 16.4942 6 1.93365 3.85515 2014-07-01 16:44:00,578.578 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1674 16 6916 6900 16.4843 0 - 3.85515 2014-07-01 16:44:00,765.765 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:01,509.509 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:01,577.577 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1675 16 6916 6900 16.4745 0 - 3.85515 2014-07-01 16:44:02,578.578 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1676 16 6916 6900 16.4647 0 - 3.85515 2014-07-01 16:44:03,068.068 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:03,578.578 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1677 16 6916 6900 16.4549 0 - 3.85515 2014-07-01 16:44:04,578.578 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1678 16 6916 6900 16.445 0 - 3.85515 2014-07-01 16:44:05,580.580 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1679 16 6916 6900 16.4353 0 - 3.85515 2014-07-01 16:44:06,380.380 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:43:11.319249', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886687812, u'health': u'HEALTH_OK', u'kb_used': 10093240, u'store_stats': {u'bytes_total': 4682739, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2503/15118 objects degraded (16.556%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2503, u'num_pgs': 244, u'data_bytes': 29955927851, u'degraded_total': 15118, u'bytes_used': 58002685952, u'op_per_sec': 0, u'write_bytes_sec': 2106845, u'version': 1378, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'degraded_ratio': u'16.556', u'bytes_avail': 4940579852288}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:06,381.381 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:06,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:44:06.491478min lat: 0.057269 max lat: 294.525 avg lat: 3.85515 2014-07-01 16:44:06,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:44:06,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1680 16 6916 6900 16.4255 0 - 3.85515 2014-07-01 16:44:06,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 2 to mira0435601-400 2014-07-01 16:44:06,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 4 to mira0435601-400 2014-07-01 16:44:07,095.095 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:44:07,096.096 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:07,579.579 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1681 16 6919 6903 16.4228 1.5 7.94111 3.85938 2014-07-01 16:44:08,054.054 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:08,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 5 to mira0435601-400 2014-07-01 16:44:08,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 216 (ObjNum 1226 snap 0 seq_num 1226) dirty exists 2014-07-01 16:44:08,617.617 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1682 16 6923 6907 16.4226 16 0.102471 3.86092 2014-07-01 16:44:09,612.612 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1683 16 6923 6907 16.4128 0 - 3.86092 2014-07-01 16:44:09,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: finishing write tid 1 to mira0435601-475 2014-07-01 16:44:09,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: finishing write tid 3 to mira0435601-475 2014-07-01 16:44:09,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: finishing write tid 4 to mira0435601-467 2014-07-01 16:44:09,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 127 (ObjNum 1221 snap 0 seq_num 1221) dirty exists 2014-07-01 16:44:10,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: done (15 left) 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: done (14 left) 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: done (13 left) 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: done (12 left) 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1844: done (11 left) 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 216 current snap is 0 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: seq_num 1234 ranges {1191936=757760,1949696=512000,2461696=446464,2908160=245760} 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: writing mira0435601-216 from 1191936 to 1949696 tid 1 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: writing mira0435601-216 from 1949696 to 2461696 tid 2 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: writing mira0435601-216 from 2461696 to 2908160 tid 3 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: writing mira0435601-216 from 2908160 to 3153920 tid 4 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1846: read oid 332 snap -1 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 1 to mira0435601-461 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 2 to mira0435601-461 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 3 to mira0435601-461 2014-07-01 16:44:10,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: expect (ObjNum 384 snap 0 seq_num 384) 2014-07-01 16:44:10,580.580 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1684 16 6939 6923 16.4411 32 0.506109 3.8814 2014-07-01 16:44:11,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 4 to mira0435601-461 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 1 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 2 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 3 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 4 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 6 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 7 to mira0435601-216 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 216 v 240 (ObjNum 1234 snap 0 seq_num 1234) dirty exists 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: done (12 left) 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: done (11 left) 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 16:44:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: seq_num 1235 ranges {4259840=782336} 2014-07-01 16:44:11,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: writing mira0435601-409 from 4259840 to 5042176 tid 1 2014-07-01 16:44:11,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 194 current snap is 0 2014-07-01 16:44:11,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: seq_num 1236 ranges {6787072=458752} 2014-07-01 16:44:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: writing mira0435601-194 from 6787072 to 7245824 tid 1 2014-07-01 16:44:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: read oid 207 snap -1 2014-07-01 16:44:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: read oid 234 snap -1 2014-07-01 16:44:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: read oid 454 snap -1 2014-07-01 16:44:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:44:11,581.581 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1685 16 6944 6928 16.4432 20 0.6901 3.87891 2014-07-01 16:44:12,580.580 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1686 16 6944 6928 16.4334 0 - 3.87891 2014-07-01 16:44:12,602.602 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:13,338.338 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:13,580.580 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1687 16 6944 6928 16.4237 0 - 3.87891 2014-07-01 16:44:14,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1688 16 6944 6928 16.414 0 - 3.87891 2014-07-01 16:44:14,882.882 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:15,581.581 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1689 16 6944 6928 16.4042 0 - 3.87891 2014-07-01 16:44:15,619.619 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2504/15182 objects degraded (16.493%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 11329710, u'degraded_objects': 2504, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 30094384939, u'degraded_total': 15182, u'bytes_used': 58412253184, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 24408388, u'op_per_sec': 11, u'version': 1384, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 525548, u'degraded_ratio': u'16.493', u'bytes_avail': 4940170285056}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:15,619.619 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:16,395.395 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:44:16,395.395 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:16,581.581 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1690 16 6944 6928 16.3945 0 - 3.87891 2014-07-01 16:44:17,184.184 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:17,581.581 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1691 16 6944 6928 16.3848 0 - 3.87891 2014-07-01 16:44:18,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1692 16 6944 6928 16.3752 0 - 3.87891 2014-07-01 16:44:19,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1693 16 6944 6928 16.3655 0 - 3.87891 2014-07-01 16:44:20,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1694 16 6944 6928 16.3558 0 - 3.87891 2014-07-01 16:44:21,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1695 16 6944 6928 16.3462 0 - 3.87891 2014-07-01 16:44:21,743.743 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:22,506.506 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:22,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1696 16 6944 6928 16.3365 0 - 3.87891 2014-07-01 16:44:23,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1697 16 6944 6928 16.3269 0 - 3.87891 2014-07-01 16:44:24,066.066 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:24,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1698 16 6944 6928 16.3173 0 - 3.87891 2014-07-01 16:44:24,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: expect (ObjNum 980 snap 0 seq_num 980) 2014-07-01 16:44:24,768.768 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2504/15182 objects degraded (16.493%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 11329710, u'degraded_objects': 2504, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 30094384939, u'degraded_total': 15182, u'bytes_used': 58412253184, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 24408388, u'op_per_sec': 11, u'version': 1384, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 525548, u'degraded_ratio': u'16.493', u'bytes_avail': 4940170285056}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:24,768.768 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:25,535.535 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:44:25,535.535 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:25,582.582 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1699 16 6944 6928 16.3077 0 - 3.87891 2014-07-01 16:44:26,348.348 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:26,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:44:26.495610min lat: 0.057269 max lat: 294.525 avg lat: 3.87891 2014-07-01 16:44:26,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:44:26,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1700 16 6944 6928 16.2981 0 - 3.87891 2014-07-01 16:44:27,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: done (15 left) 2014-07-01 16:44:27,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: read oid 426 snap -1 2014-07-01 16:44:27,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:44:27,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: expect (ObjNum 425 snap 0 seq_num 425) 2014-07-01 16:44:27,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1701 16 6944 6928 16.2885 0 - 3.87891 2014-07-01 16:44:28,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: done (15 left) 2014-07-01 16:44:28,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1853: read oid 451 snap -1 2014-07-01 16:44:28,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:44:28,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1702 16 6944 6928 16.2789 0 - 3.87891 2014-07-01 16:44:29,583.583 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1703 16 6944 6928 16.2694 0 - 3.87891 2014-07-01 16:44:30,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1704 16 6944 6928 16.2598 0 - 3.87891 2014-07-01 16:44:30,734.734 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:31,508.508 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:31,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1705 16 6944 6928 16.2503 0 - 3.87891 2014-07-01 16:44:32,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 6 to mira0435601-461 2014-07-01 16:44:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: finishing write tid 4 to mira0435601-475 2014-07-01 16:44:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 219 (ObjNum 1222 snap 0 seq_num 1222) dirty exists 2014-07-01 16:44:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: done (15 left) 2014-07-01 16:44:32,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 274 current snap is 0 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: seq_num 1237 ranges {0=577536,577536=753664,1331200=49152} 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: writing mira0435601-274 from 0 to 577536 tid 1 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: writing mira0435601-274 from 577536 to 1331200 tid 2 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: writing mira0435601-274 from 1331200 to 1380352 tid 3 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: finishing write tid 7 to mira0435601-461 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 106 (ObjNum 1225 snap 0 seq_num 1225) dirty exists 2014-07-01 16:44:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1853: expect (ObjNum 1203 snap 0 seq_num 1203) 2014-07-01 16:44:32,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1706 16 6948 6932 16.2502 0.761905 48.0938 3.89901 2014-07-01 16:44:32,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:1832: expect (ObjNum 932 snap 0 seq_num 932) 2014-07-01 16:44:33,051.051 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:33,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1707 16 6948 6932 16.2406 0 - 3.89901 2014-07-01 16:44:33,794.794 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'147 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2498/15194 objects degraded (16.441%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5171493, u'degraded_objects': 2498, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 30124711723, u'degraded_total': 15194, u'bytes_used': 58705829888, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 7478465, u'op_per_sec': 7, u'version': 1388, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1474431, u'degraded_ratio': u'16.441', u'bytes_avail': 4939876708352}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:33,794.794 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:34,570.570 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:44:34,570.570 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:34,584.584 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1708 16 6960 6944 16.2592 24 23.0967 3.93369 2014-07-01 16:44:34,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 1 to mira0435601-359 2014-07-01 16:44:34,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 2 to mira0435601-359 2014-07-01 16:44:34,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1842: expect (ObjNum 184 snap 0 seq_num 184) 2014-07-01 16:44:34,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1846: expect (ObjNum 1158 snap 0 seq_num 1158) 2014-07-01 16:44:34,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: expect (ObjNum 453 snap 0 seq_num 453) 2014-07-01 16:44:35,381.381 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:35,588.588 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1709 16 6971 6955 16.2754 44 1.29809 3.93031 2014-07-01 16:44:35,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 3 to mira0435601-359 2014-07-01 16:44:35,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 5 to mira0435601-359 2014-07-01 16:44:35,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 1 to mira0435601-235 2014-07-01 16:44:35,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 2 to mira0435601-235 2014-07-01 16:44:35,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1832: done (15 left) 2014-07-01 16:44:35,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: done (14 left) 2014-07-01 16:44:35,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1842: done (13 left) 2014-07-01 16:44:35,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1846: done (12 left) 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: done (11 left) 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1853: done (10 left) 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 388 current snap is 0 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: seq_num 1238 ranges {835584=200704} 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: writing mira0435601-388 from 835584 to 1036288 tid 1 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1856: read oid 253 snap -1 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 3 to mira0435601-235 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 5 to mira0435601-235 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 6 to mira0435601-235 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 176 (ObjNum 1227 snap 0 seq_num 1227) dirty exists 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: finishing write tid 1 to mira0435601-409 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: finishing write tid 3 to mira0435601-409 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 1 to mira0435601-194 2014-07-01 16:44:35,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 3 to mira0435601-194 2014-07-01 16:44:35,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: finishing write tid 1 to mira0435601-222 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: finishing write tid 3 to mira0435601-222 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 1 to mira0435601-496 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 2 to mira0435601-496 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 3 to mira0435601-496 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 4 to mira0435601-496 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 6 to mira0435601-496 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 1 to mira0435601-449 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 2 to mira0435601-449 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 4 to mira0435601-449 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 1 to mira0435601-458 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 2 to mira0435601-458 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 3 to mira0435601-458 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 4 to mira0435601-458 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 6 to mira0435601-458 2014-07-01 16:44:36,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: finishing write tid 4 to mira0435601-222 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 221 (ObjNum 1229 snap 0 seq_num 1229) dirty exists 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 6 to mira0435601-359 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 186 (ObjNum 1228 snap 0 seq_num 1228) dirty exists 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 7 to mira0435601-496 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 131 (ObjNum 1230 snap 0 seq_num 1230) dirty exists 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 5 to mira0435601-449 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 224 (ObjNum 1231 snap 0 seq_num 1231) dirty exists 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: finishing write tid 4 to mira0435601-409 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 245 (ObjNum 1235 snap 0 seq_num 1235) dirty exists 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: finishing write tid 1 to mira0435601-274 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: finishing write tid 2 to mira0435601-274 2014-07-01 16:44:36,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: done (11 left) 2014-07-01 16:44:36,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: done (10 left) 2014-07-01 16:44:36,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: done (9 left) 2014-07-01 16:44:36,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: done (8 left) 2014-07-01 16:44:36,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: done (7 left) 2014-07-01 16:44:36,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: done (6 left) 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: done (5 left) 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1857: read oid 222 snap -1 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: finishing write tid 3 to mira0435601-274 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: finishing write tid 5 to mira0435601-274 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: finishing write tid 7 to mira0435601-458 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 191 (ObjNum 1232 snap 0 seq_num 1232) dirty exists 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: done (5 left) 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1858: read oid 165 snap -1 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 4 to mira0435601-194 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 245 (ObjNum 1236 snap 0 seq_num 1236) dirty exists 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: done (5 left) 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1859: read oid 331 snap -1 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 149 current snap is 0 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: seq_num 1239 ranges {3362816=630784,3993600=516096,4509696=286720} 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: writing mira0435601-149 from 3362816 to 3993600 tid 1 2014-07-01 16:44:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: writing mira0435601-149 from 3993600 to 4509696 tid 2 2014-07-01 16:44:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: writing mira0435601-149 from 4509696 to 4796416 tid 3 2014-07-01 16:44:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: read oid 140 snap -1 2014-07-01 16:44:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1857: expect (ObjNum 1229 snap 0 seq_num 1229) 2014-07-01 16:44:36,696.696 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1710 16 6974 6958 16.2729 12 0.464651 3.92882 2014-07-01 16:44:37,585.585 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1711 16 6987 6971 16.2938 52 1.83075 3.92573 2014-07-01 16:44:38,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1856: expect (ObjNum 804 snap 0 seq_num 804) 2014-07-01 16:44:38,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1712 16 6991 6975 16.2937 16 2.93364 3.92458 2014-07-01 16:44:39,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1713 16 6993 6977 16.2888 8 0.41087 3.92377 2014-07-01 16:44:39,936.936 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:40,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1856: done (7 left) 2014-07-01 16:44:40,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1857: done (6 left) 2014-07-01 16:44:40,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1859: done (5 left) 2014-07-01 16:44:40,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 171 current snap is 0 2014-07-01 16:44:40,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: seq_num 1240 ranges {1798144=798720} 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: writing mira0435601-171 from 1798144 to 2596864 tid 1 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: seq_num 1241 ranges {0=421888,421888=733184,1155072=516096,1671168=356352} 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: writing mira0435601-336 from 0 to 421888 tid 1 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: writing mira0435601-336 from 421888 to 1155072 tid 2 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: writing mira0435601-336 from 1155072 to 1671168 tid 3 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: writing mira0435601-336 from 1671168 to 2027520 tid 4 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1864: read oid 94 snap -1 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 79 current snap is 0 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: seq_num 1242 ranges {1945600=450560,2396160=401408} 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: writing mira0435601-79 from 1945600 to 2396160 tid 1 2014-07-01 16:44:40,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: writing mira0435601-79 from 2396160 to 2797568 tid 2 2014-07-01 16:44:40,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1866: delete oid 343 current snap is 0 2014-07-01 16:44:40,628.628 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1714 16 6994 6978 16.2816 4 1.29369 3.92339 2014-07-01 16:44:41,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1715 16 6994 6978 16.2721 0 - 3.92339 2014-07-01 16:44:42,358.358 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:42,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1716 16 6994 6978 16.2627 0 - 3.92339 2014-07-01 16:44:43,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1717 16 6994 6978 16.2532 0 - 3.92339 2014-07-01 16:44:43,900.900 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:44,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: finishing write tid 6 to mira0435601-274 2014-07-01 16:44:44,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 131 (ObjNum 1237 snap 0 seq_num 1237) dirty exists 2014-07-01 16:44:44,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: finishing write tid 1 to mira0435601-388 2014-07-01 16:44:44,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: finishing write tid 3 to mira0435601-388 2014-07-01 16:44:44,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: finishing write tid 1 to mira0435601-149 2014-07-01 16:44:44,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: finishing write tid 2 to mira0435601-149 2014-07-01 16:44:44,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: finishing write tid 3 to mira0435601-149 2014-07-01 16:44:44,586.586 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1718 16 6996 6980 16.2484 2 5.53759 3.92408 2014-07-01 16:44:44,671.671 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:43:44.711331', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883055576, u'health': u'HEALTH_OK', u'kb_used': 13725476, u'store_stats': {u'bytes_total': 5731779, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2505/15285 objects degraded (16.389%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2505, u'num_pgs': 244, u'data_bytes': 30311006010, u'degraded_total': 15285, u'bytes_used': 59277791232, u'op_per_sec': 3, u'write_bytes_sec': 14623354, u'version': 1395, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'degraded_ratio': u'16.389', u'bytes_avail': 4939304747008}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:44,671.671 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:45,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: finishing write tid 5 to mira0435601-149 2014-07-01 16:44:45,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:1858: expect (ObjNum 1148 snap 0 seq_num 1148) 2014-07-01 16:44:45,484.484 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:44:45,484.484 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:45,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1719 16 7012 6996 16.2762 64 0.245288 3.9307 2014-07-01 16:44:46,281.281 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:46,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:44:46.498999min lat: 0.057269 max lat: 294.525 avg lat: 3.92588 2014-07-01 16:44:46,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:44:46,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1720 16 7022 7006 16.2899 40 0.702731 3.92588 2014-07-01 16:44:47,588.588 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1721 16 7024 7008 16.2851 8 1.34729 3.92608 2014-07-01 16:44:47,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1864: expect (ObjNum 1138 snap 0 seq_num 1138) 2014-07-01 16:44:48,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1722 16 7024 7008 16.2757 0 - 3.92608 2014-07-01 16:44:49,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1723 16 7029 7013 16.2778 10 0.122098 3.92502 2014-07-01 16:44:50,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1724 16 7029 7013 16.2684 0 - 3.92502 2014-07-01 16:44:50,806.806 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:51,568.568 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:51,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1725 16 7029 7013 16.259 0 - 3.92502 2014-07-01 16:44:52,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1726 16 7029 7013 16.2495 0 - 3.92502 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: finishing write tid 6 to mira0435601-149 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 215 (ObjNum 1239 snap 0 seq_num 1239) dirty exists 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: finishing write tid 4 to mira0435601-388 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 193 (ObjNum 1238 snap 0 seq_num 1238) dirty exists 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: finishing write tid 1 to mira0435601-79 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 1 to mira0435601-336 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: finishing write tid 2 to mira0435601-79 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 2 to mira0435601-336 2014-07-01 16:44:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: finishing write tid 4 to mira0435601-79 2014-07-01 16:44:53,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 3 to mira0435601-336 2014-07-01 16:44:53,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 4 to mira0435601-336 2014-07-01 16:44:53,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 6 to mira0435601-336 2014-07-01 16:44:53,127.127 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:53,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: expect (ObjNum 1076 snap 0 seq_num 1076) 2014-07-01 16:44:53,587.587 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1727 16 7043 7027 16.2726 14 0.198099 3.928 2014-07-01 16:44:53,824.824 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:44:44.711707', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883044132, u'health': u'HEALTH_OK', u'kb_used': 13736920, u'store_stats': {u'bytes_total': 6780355, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:43:54.536698', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886682300, u'health': u'HEALTH_OK', u'kb_used': 10098752, u'store_stats': {u'bytes_total': 5731273, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2510/15368 objects degraded (16.333%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2510, u'num_pgs': 244, u'data_bytes': 30479097658, u'degraded_total': 15368, u'bytes_used': 59602739200, u'op_per_sec': 1, u'write_bytes_sec': 8115070, u'version': 1400, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'degraded_ratio': u'16.333', u'bytes_avail': 4938979799040}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:44:53,824.824 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:44:54,588.588 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1728 16 7046 7030 16.2701 12 0.970131 3.92763 2014-07-01 16:44:54,600.600 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:44:54,600.600 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:44:55,419.419 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:44:55,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1729 16 7046 7030 16.2607 0 - 3.92763 2014-07-01 16:44:56,590.590 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1730 16 7046 7030 16.2513 0 - 3.92763 2014-07-01 16:44:56,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: finishing write tid 1 to mira0435601-171 2014-07-01 16:44:56,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: finishing write tid 3 to mira0435601-171 2014-07-01 16:44:56,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: finishing write tid 7 to mira0435601-336 2014-07-01 16:44:56,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 136 (ObjNum 1241 snap 0 seq_num 1241) dirty exists 2014-07-01 16:44:56,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: finishing write tid 5 to mira0435601-79 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 227 (ObjNum 1242 snap 0 seq_num 1242) dirty exists 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: finishing write tid 4 to mira0435601-171 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 218 (ObjNum 1240 snap 0 seq_num 1240) dirty exists 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: done (9 left) 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: done (8 left) 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1858: done (7 left) 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: done (6 left) 2014-07-01 16:44:56,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: done (5 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: done (4 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: done (3 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1864: done (2 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: done (1 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1866: done (0 left) 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1867: read oid 230 snap -1 2014-07-01 16:44:56,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:1868: delete oid 477 current snap is 0 2014-07-01 16:44:57,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1731 16 7046 7030 16.2419 0 - 3.92763 2014-07-01 16:44:58,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1732 16 7046 7030 16.2325 0 - 3.92763 2014-07-01 16:44:59,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1733 16 7046 7030 16.2231 0 - 3.92763 2014-07-01 16:44:59,804.804 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:00,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1734 16 7046 7030 16.2138 0 - 3.92763 2014-07-01 16:45:00,999.999 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:01,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1868: done (1 left) 2014-07-01 16:45:01,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: delete oid 418 current snap is 0 2014-07-01 16:45:01,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: done (1 left) 2014-07-01 16:45:01,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 386 current snap is 0 2014-07-01 16:45:01,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: seq_num 1246 ranges {0=139264} 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: writing mira0435601-386 from 0 to 139264 tid 1 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: read oid 139 snap -1 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1872: read oid 242 snap -1 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: read oid 39 snap -1 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 279 current snap is 0 2014-07-01 16:45:01,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: seq_num 1247 ranges {2711552=12288} 2014-07-01 16:45:01,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: writing mira0435601-279 from 2711552 to 2723840 tid 1 2014-07-01 16:45:01,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1875: delete oid 32 current snap is 0 2014-07-01 16:45:01,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1872: expect (ObjNum 742 snap 0 seq_num 742) 2014-07-01 16:45:01,660.660 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1735 16 7057 7041 16.2298 6.28571 7.28996 3.93699 2014-07-01 16:45:02,592.592 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1736 16 7072 7056 16.255 60 0.245966 3.93632 2014-07-01 16:45:02,747.747 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:02,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: finishing write tid 1 to mira0435601-386 2014-07-01 16:45:02,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: finishing write tid 3 to mira0435601-386 2014-07-01 16:45:03,480.480 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:44:44.711707', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883044132, u'health': u'HEALTH_OK', u'kb_used': 13736920, u'store_stats': {u'bytes_total': 6780355, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:44:54.536969', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886674488, u'health': u'HEALTH_OK', u'kb_used': 10106564, u'store_stats': {u'bytes_total': 7828425, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:44:11.319617', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886680416, u'health': u'HEALTH_OK', u'kb_used': 10100636, u'store_stats': {u'bytes_total': 6779891, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2512/15395 objects degraded (16.317%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2512, u'num_pgs': 244, u'data_bytes': 30538583866, u'degraded_total': 15395, u'bytes_used': 59980161024, u'op_per_sec': 1, u'write_bytes_sec': 4363883, u'version': 1405, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 355, u'degraded_ratio': u'16.317', u'bytes_avail': 4938602377216}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:03,481.481 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:03,589.589 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1737 16 7072 7056 16.2456 0 - 3.93632 2014-07-01 16:45:03,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: finishing write tid 4 to mira0435601-386 2014-07-01 16:45:03,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 247 (ObjNum 1246 snap 0 seq_num 1246) dirty exists 2014-07-01 16:45:04,185.185 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:45:04,185.185 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:04,590.590 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1738 16 7076 7060 16.2455 8 1.31454 3.93484 2014-07-01 16:45:04,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1867: expect (ObjNum 1032 snap 0 seq_num 1032) 2014-07-01 16:45:05,026.026 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:05,640.640 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1739 16 7086 7070 16.2592 40 4.21161 3.93209 2014-07-01 16:45:06,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:1867: done (6 left) 2014-07-01 16:45:06,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: done (5 left) 2014-07-01 16:45:06,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: done (4 left) 2014-07-01 16:45:06,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:1872: done (3 left) 2014-07-01 16:45:06,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:1875: done (2 left) 2014-07-01 16:45:06,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 261 current snap is 0 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: seq_num 1249 ranges {2732032=761856,3493888=540672,4034560=405504,4440064=258048} 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: writing mira0435601-261 from 2732032 to 3493888 tid 1 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: writing mira0435601-261 from 3493888 to 4034560 tid 2 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: writing mira0435601-261 from 4034560 to 4440064 tid 3 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: writing mira0435601-261 from 4440064 to 4698112 tid 4 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 139 current snap is 0 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: seq_num 1250 ranges {0=634880,634880=524288,1159168=679936} 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: writing mira0435601-139 from 0 to 634880 tid 1 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: writing mira0435601-139 from 634880 to 1159168 tid 2 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: writing mira0435601-139 from 1159168 to 1839104 tid 3 2014-07-01 16:45:06,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 437 current snap is 0 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: seq_num 1251 ranges {0=663552} 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: writing mira0435601-437 from 0 to 663552 tid 1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: read oid 55 snap -1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1880: read oid 196 snap -1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: read oid 209 snap -1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1882: read oid 270 snap -1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 404 current snap is 0 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: seq_num 1252 ranges {0=532480} 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: writing mira0435601-404 from 0 to 532480 tid 1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1884: read oid 155 snap -1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1880: done (10 left) 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1882: done (9 left) 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 102 current snap is 0 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: seq_num 1253 ranges {2404352=405504} 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: writing mira0435601-102 from 2404352 to 2809856 tid 1 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 271 current snap is 0 2014-07-01 16:45:06,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: seq_num 1254 ranges {0=81920} 2014-07-01 16:45:06,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: writing mira0435601-271 from 0 to 81920 tid 1 2014-07-01 16:45:06,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1887: read oid 392 snap -1 2014-07-01 16:45:06,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: expect (ObjNum 526 snap 0 seq_num 526) 2014-07-01 16:45:06,764.764 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:45:06.502317min lat: 0.057269 max lat: 294.525 avg lat: 3.93209 2014-07-01 16:45:06,764.764 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:45:06,764.764 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1740 16 7086 7070 16.2498 0 - 3.93209 2014-07-01 16:45:07,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1884: expect (ObjNum 1176 snap 0 seq_num 1176) 2014-07-01 16:45:07,590.590 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1741 16 7086 7070 16.2405 0 - 3.93209 2014-07-01 16:45:07,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:1887: expect (ObjNum 1181 snap 0 seq_num 1181) 2014-07-01 16:45:08,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: done (11 left) 2014-07-01 16:45:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1884: done (10 left) 2014-07-01 16:45:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1887: done (9 left) 2014-07-01 16:45:08,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: read oid 386 snap -1 2014-07-01 16:45:08,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1889: read oid 417 snap -1 2014-07-01 16:45:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1890: delete oid 403 current snap is 0 2014-07-01 16:45:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: expect (ObjNum 1246 snap 0 seq_num 1246) 2014-07-01 16:45:08,603.603 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1742 16 7086 7070 16.2312 0 - 3.93209 2014-07-01 16:45:09,627.627 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1743 16 7086 7070 16.2218 0 - 3.93209 2014-07-01 16:45:09,767.767 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:10,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: expect (ObjNum 38 snap 0 seq_num 38) 2014-07-01 16:45:10,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: expect (ObjNum 703 snap 0 seq_num 703) 2014-07-01 16:45:10,651.651 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1744 16 7087 7071 16.2148 0.8 8.91853 3.9328 2014-07-01 16:45:10,705.705 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:11,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1745 16 7088 7072 16.2078 4 6.04961 3.9331 2014-07-01 16:45:12,254.254 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:12,590.590 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1746 16 7088 7072 16.1985 0 - 3.9331 2014-07-01 16:45:12,984.984 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.027704', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.013033', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:44:44.711707', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883044132, u'health': u'HEALTH_OK', u'kb_used': 13736920, u'store_stats': {u'bytes_total': 6780355, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:44:54.536969', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886674488, u'health': u'HEALTH_OK', u'kb_used': 10106564, u'store_stats': {u'bytes_total': 7828425, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2518/15465 objects degraded (16.282%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2518, u'num_pgs': 244, u'data_bytes': 30689333050, u'degraded_total': 15465, u'bytes_used': 60204789760, u'op_per_sec': 7, u'write_bytes_sec': 11142791, u'version': 1411, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1238805, u'degraded_ratio': u'16.282', u'bytes_avail': 4938377748480}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:12,985.985 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:13,591.591 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1747 16 7088 7072 16.1893 0 - 3.9331 2014-07-01 16:45:13,758.758 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:45:13,758.758 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:14,591.591 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1748 16 7088 7072 16.18 0 - 3.9331 2014-07-01 16:45:15,592.592 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1749 16 7088 7072 16.1708 0 - 3.9331 2014-07-01 16:45:16,592.592 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1750 16 7088 7072 16.1615 0 - 3.9331 2014-07-01 16:45:17,592.592 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1751 16 7088 7072 16.1523 0 - 3.9331 2014-07-01 16:45:18,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1752 16 7088 7072 16.1431 0 - 3.9331 2014-07-01 16:45:19,206.206 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: done (11 left) 2014-07-01 16:45:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: done (10 left) 2014-07-01 16:45:19,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: done (9 left) 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1889: done (8 left) 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1890: done (7 left) 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: read oid 280 snap -1 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: finishing write tid 1 to mira0435601-102 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1892: read oid 471 snap -1 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: finishing write tid 3 to mira0435601-102 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 347 current snap is 0 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: seq_num 1256 ranges {1789952=782336,2572288=401408} 2014-07-01 16:45:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: writing mira0435601-347 from 1789952 to 2572288 tid 1 2014-07-01 16:45:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: writing mira0435601-347 from 2572288 to 2973696 tid 2 2014-07-01 16:45:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: delete oid 303 current snap is 0 2014-07-01 16:45:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:1892: expect (ObjNum 1160 snap 0 seq_num 1160) 2014-07-01 16:45:19,712.712 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1753 16 7094 7078 16.1476 3 9.24598 3.94169 2014-07-01 16:45:20,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1754 16 7107 7091 16.168 52 1.69426 3.95453 2014-07-01 16:45:21,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1755 16 7116 7100 16.1793 36 0.726516 3.95368 2014-07-01 16:45:22,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1756 16 7118 7102 16.1746 8 2.4369 3.95312 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 1 to mira0435601-437 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 3 to mira0435601-437 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 1 to mira0435601-404 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 1 to mira0435601-261 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 3 to mira0435601-404 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 2 to mira0435601-261 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 3 to mira0435601-261 2014-07-01 16:45:23,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 4 to mira0435601-261 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: finishing write tid 1 to mira0435601-271 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 6 to mira0435601-261 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: finishing write tid 3 to mira0435601-271 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: finishing write tid 1 to mira0435601-279 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: finishing write tid 3 to mira0435601-279 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: finishing write tid 1 to mira0435601-139 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: finishing write tid 2 to mira0435601-139 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: finishing write tid 3 to mira0435601-139 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: finishing write tid 5 to mira0435601-139 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1892: done (10 left) 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: done (9 left) 2014-07-01 16:45:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 251 current snap is 0 2014-07-01 16:45:23,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: seq_num 1258 ranges {1925120=573440,2498560=483328,2981888=577536,3559424=143360} 2014-07-01 16:45:23,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: writing mira0435601-251 from 1925120 to 2498560 tid 1 2014-07-01 16:45:23,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: writing mira0435601-251 from 2498560 to 2981888 tid 2 2014-07-01 16:45:23,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: writing mira0435601-251 from 2981888 to 3559424 tid 3 2014-07-01 16:45:23,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: writing mira0435601-251 from 3559424 to 3702784 tid 4 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 463 current snap is 0 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: seq_num 1259 ranges {1130496=602112} 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: writing mira0435601-463 from 1130496 to 1732608 tid 1 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 392 current snap is 0 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: seq_num 1260 ranges {1712128=475136} 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: writing mira0435601-392 from 1712128 to 2187264 tid 1 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1898: delete oid 136 current snap is 0 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: finishing write tid 1 to mira0435601-347 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: finishing write tid 2 to mira0435601-347 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: finishing write tid 4 to mira0435601-347 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: finishing write tid 4 to mira0435601-102 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 230 (ObjNum 1253 snap 0 seq_num 1253) dirty exists 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: finishing write tid 5 to mira0435601-347 2014-07-01 16:45:23,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 219 (ObjNum 1256 snap 0 seq_num 1256) dirty exists 2014-07-01 16:45:23,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1757 16 7119 7103 16.1677 4 2.54991 3.95293 2014-07-01 16:45:23,753.753 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:24,593.593 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1758 16 7119 7103 16.1585 0 - 3.95293 2014-07-01 16:45:25,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1759 16 7119 7103 16.1493 0 - 3.95293 2014-07-01 16:45:26,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: expect (ObjNum 902 snap 0 seq_num 902) 2014-07-01 16:45:26,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:45:26.506102min lat: 0.057269 max lat: 294.525 avg lat: 3.95386 2014-07-01 16:45:26,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:45:26,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1760 16 7123 7107 16.1492 5.33333 5.3622 3.95386 2014-07-01 16:45:26,830.830 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: finishing write tid 7 to mira0435601-261 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 221 (ObjNum 1249 snap 0 seq_num 1249) dirty exists 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: finishing write tid 4 to mira0435601-271 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 271 v 178 (ObjNum 1254 snap 0 seq_num 1254) dirty exists 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 4 to mira0435601-437 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 176 (ObjNum 1251 snap 0 seq_num 1251) dirty exists 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: done (12 left) 2014-07-01 16:45:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: done (11 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: done (10 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: done (9 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: done (8 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: done (7 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1898: done (6 left) 2014-07-01 16:45:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1899: delete oid 112 current snap is 0 2014-07-01 16:45:27,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1761 16 7124 7108 16.1423 4 5.80663 3.95412 2014-07-01 16:45:28,386.386 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:28,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1762 16 7124 7108 16.1332 0 - 3.95412 2014-07-01 16:45:29,105.105 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:44:44.711707', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883044132, u'health': u'HEALTH_OK', u'kb_used': 13736920, u'store_stats': {u'bytes_total': 6780355, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:44:54.536969', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886674488, u'health': u'HEALTH_OK', u'kb_used': 10106564, u'store_stats': {u'bytes_total': 7828425, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'17 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2519/15545 objects degraded (16.205%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2519, u'num_pgs': 244, u'data_bytes': 30846381882, u'degraded_total': 15545, u'bytes_used': 60465389568, u'op_per_sec': 3, u'write_bytes_sec': 6264922, u'version': 1417, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 202, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'read_bytes_sec': 433291, u'degraded_ratio': u'16.205', u'bytes_avail': 4938117148672}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:29,105.105 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:29,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: finishing write tid 1 to mira0435601-463 2014-07-01 16:45:29,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: finishing write tid 3 to mira0435601-463 2014-07-01 16:45:29,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: finishing write tid 1 to mira0435601-392 2014-07-01 16:45:29,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: finishing write tid 3 to mira0435601-392 2014-07-01 16:45:29,594.594 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1763 16 7136 7120 16.1512 24 2.17194 3.95776 2014-07-01 16:45:29,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: finishing write tid 4 to mira0435601-279 2014-07-01 16:45:29,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 172 (ObjNum 1247 snap 0 seq_num 1247) dirty exists 2014-07-01 16:45:29,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: finishing write tid 6 to mira0435601-139 2014-07-01 16:45:29,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 139 v 174 (ObjNum 1250 snap 0 seq_num 1250) dirty exists 2014-07-01 16:45:29,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 1 to mira0435601-251 2014-07-01 16:45:29,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 2 to mira0435601-251 2014-07-01 16:45:29,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 3 to mira0435601-251 2014-07-01 16:45:29,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 4 to mira0435601-251 2014-07-01 16:45:29,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 6 to mira0435601-251 2014-07-01 16:45:29,889.889 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:45:29,889.889 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:29,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: finishing write tid 4 to mira0435601-463 2014-07-01 16:45:29,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 109 (ObjNum 1259 snap 0 seq_num 1259) dirty exists 2014-07-01 16:45:30,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1764 16 7151 7135 16.1761 60 0.355149 3.95327 2014-07-01 16:45:30,722.722 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:31,595.595 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1765 16 7152 7136 16.1692 4 1.35728 3.9529 2014-07-01 16:45:32,596.596 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1766 16 7152 7136 16.16 0 - 3.9529 2014-07-01 16:45:33,596.596 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1767 16 7152 7136 16.1509 0 - 3.9529 2014-07-01 16:45:33,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: finishing write tid 4 to mira0435601-392 2014-07-01 16:45:33,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 392 v 229 (ObjNum 1260 snap 0 seq_num 1260) dirty exists 2014-07-01 16:45:33,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 4 to mira0435601-404 2014-07-01 16:45:33,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 195 (ObjNum 1252 snap 0 seq_num 1252) dirty exists 2014-07-01 16:45:34,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 7 to mira0435601-251 2014-07-01 16:45:34,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 177 (ObjNum 1258 snap 0 seq_num 1258) dirty exists 2014-07-01 16:45:34,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1768 16 7159 7143 16.1576 9.33333 0.08393 3.95142 2014-07-01 16:45:35,125.125 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:35,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1769 16 7159 7143 16.1484 0 - 3.95142 2014-07-01 16:45:35,899.899 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:36,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1770 16 7159 7143 16.1393 0 - 3.95142 2014-07-01 16:45:37,533.533 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:37,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1771 16 7159 7143 16.1302 0 - 3.95142 2014-07-01 16:45:38,334.334 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:44:44.711707', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883044132, u'health': u'HEALTH_OK', u'kb_used': 13736920, u'store_stats': {u'bytes_total': 6780355, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:44:54.536969', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886674488, u'health': u'HEALTH_OK', u'kb_used': 10106564, u'store_stats': {u'bytes_total': 7828425, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'41 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2476/15602 objects degraded (15.870%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2476, u'num_pgs': 244, u'data_bytes': 30970642234, u'degraded_total': 15602, u'bytes_used': 60860358656, u'op_per_sec': 1, u'write_bytes_sec': 1050857, u'version': 1423, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 203, u'state_name': u'active+clean'}, {u'count': 19, u'state_name': u'active+recovering'}], u'read_bytes_sec': 769, u'degraded_ratio': u'15.870', u'bytes_avail': 4937722179584}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:38,335.335 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:38,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1772 16 7159 7143 16.1211 0 - 3.95142 2014-07-01 16:45:39,212.212 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:45:39,212.212 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:39,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1773 16 7159 7143 16.112 0 - 3.95142 2014-07-01 16:45:39,908.908 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:40,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1774 16 7159 7143 16.1029 0 - 3.95142 2014-07-01 16:45:41,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1775 16 7159 7143 16.0939 0 - 3.95142 2014-07-01 16:45:42,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1776 16 7159 7143 16.0848 0 - 3.95142 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: done (6 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: done (5 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: done (4 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: done (3 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: done (2 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: done (1 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1899: done (0 left) 2014-07-01 16:45:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1900: read oid 429 snap -1 2014-07-01 16:45:42,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: read oid 3 snap -1 2014-07-01 16:45:43,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 170 current snap is 0 2014-07-01 16:45:43,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: seq_num 1263 ranges {2457600=692224,3149824=540672,3690496=450560,4141056=532480,4673536=81920} 2014-07-01 16:45:43,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: writing mira0435601-170 from 2457600 to 3149824 tid 1 2014-07-01 16:45:43,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: writing mira0435601-170 from 3149824 to 3690496 tid 2 2014-07-01 16:45:43,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: writing mira0435601-170 from 3690496 to 4141056 tid 3 2014-07-01 16:45:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: writing mira0435601-170 from 4141056 to 4673536 tid 4 2014-07-01 16:45:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: writing mira0435601-170 from 4673536 to 4755456 tid 5 2014-07-01 16:45:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1903: delete oid 485 current snap is 0 2014-07-01 16:45:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:1900: expect (ObjNum 428 snap 0 seq_num 428) 2014-07-01 16:45:43,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: expect (ObjNum 992 snap 0 seq_num 992) 2014-07-01 16:45:43,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1777 16 7181 7165 16.1253 9.77778 0.640141 3.96781 2014-07-01 16:45:44,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1778 16 7189 7173 16.1342 32 0.101635 3.96382 2014-07-01 16:45:45,070.070 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:45,597.597 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1779 16 7189 7173 16.1251 0 - 3.96382 2014-07-01 16:45:45,814.814 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:46,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:45:46.510182min lat: 0.057269 max lat: 294.525 avg lat: 3.96382 2014-07-01 16:45:46,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:45:46,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1780 16 7189 7173 16.1161 0 - 3.96382 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 1 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 2 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 3 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 4 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 5 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 7 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: finishing write tid 8 to mira0435601-170 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 173 (ObjNum 1263 snap 0 seq_num 1263) dirty exists 2014-07-01 16:45:46,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1900: done (3 left) 2014-07-01 16:45:46,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: done (2 left) 2014-07-01 16:45:46,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: done (1 left) 2014-07-01 16:45:46,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1903: done (0 left) 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1904: read oid 71 snap -1 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: read oid 260 snap -1 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: read oid 373 snap -1 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 442 current snap is 0 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: seq_num 1265 ranges {2048000=593920,2641920=401408} 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: writing mira0435601-442 from 2048000 to 2641920 tid 1 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: writing mira0435601-442 from 2641920 to 3043328 tid 2 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: read oid 413 snap -1 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: seq_num 1266 ranges {8318976=708608,9027584=692224,9719808=512000} 2014-07-01 16:45:46,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: writing mira0435601-128 from 8318976 to 9027584 tid 1 2014-07-01 16:45:46,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: writing mira0435601-128 from 9027584 to 9719808 tid 2 2014-07-01 16:45:46,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: writing mira0435601-128 from 9719808 to 10231808 tid 3 2014-07-01 16:45:46,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:1910: delete oid 363 current snap is 0 2014-07-01 16:45:47,377.377 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:47,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1781 16 7189 7173 16.107 0 - 3.96382 2014-07-01 16:45:47,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1904: expect (ObjNum 1141 snap 0 seq_num 1141) 2014-07-01 16:45:48,084.084 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:44:54.536969', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886674488, u'health': u'HEALTH_OK', u'kb_used': 10106564, u'store_stats': {u'bytes_total': 7828425, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'17 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'39 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2416/15650 objects degraded (15.438%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2416, u'num_pgs': 244, u'data_bytes': 31083802426, u'degraded_total': 15650, u'bytes_used': 61015896064, u'op_per_sec': 2, u'version': 1427, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 205, u'state_name': u'active+clean'}, {u'count': 17, u'state_name': u'active+recovering'}], u'read_bytes_sec': 68721, u'degraded_ratio': u'15.438', u'bytes_avail': 4937566642176}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:48,084.084 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:48,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1782 16 7191 7175 16.1025 2 4.83961 3.96411 2014-07-01 16:45:48,874.874 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:45:48,874.874 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:49,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1783 16 7191 7175 16.0934 0 - 3.96411 2014-07-01 16:45:49,699.699 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:49,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 1 to mira0435601-442 2014-07-01 16:45:49,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 2 to mira0435601-442 2014-07-01 16:45:49,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 4 to mira0435601-442 2014-07-01 16:45:49,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: finishing write tid 1 to mira0435601-128 2014-07-01 16:45:49,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: finishing write tid 2 to mira0435601-128 2014-07-01 16:45:49,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: finishing write tid 3 to mira0435601-128 2014-07-01 16:45:49,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: finishing write tid 5 to mira0435601-128 2014-07-01 16:45:49,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: finishing write tid 6 to mira0435601-128 2014-07-01 16:45:49,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 177 (ObjNum 1266 snap 0 seq_num 1266) dirty exists 2014-07-01 16:45:49,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: expect (ObjNum 698 snap 0 seq_num 698) 2014-07-01 16:45:50,620.620 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1784 16 7202 7186 16.1091 22 6.159 3.96716 2014-07-01 16:45:51,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1785 16 7213 7197 16.1247 44 0.508602 3.96258 2014-07-01 16:45:52,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: expect (ObjNum 1168 snap 0 seq_num 1168) 2014-07-01 16:45:52,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1786 16 7213 7197 16.1157 0 - 3.96258 2014-07-01 16:45:52,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 5 to mira0435601-442 2014-07-01 16:45:52,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 233 (ObjNum 1265 snap 0 seq_num 1265) dirty exists 2014-07-01 16:45:52,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 16:45:53,598.598 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1787 16 7213 7197 16.1066 0 - 3.96258 2014-07-01 16:45:54,282.282 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:54,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1788 16 7213 7197 16.0976 0 - 3.96258 2014-07-01 16:45:54,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1904: done (6 left) 2014-07-01 16:45:54,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: done (5 left) 2014-07-01 16:45:54,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: done (4 left) 2014-07-01 16:45:54,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: done (3 left) 2014-07-01 16:45:54,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: done (2 left) 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: done (1 left) 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1910: done (0 left) 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1911: read oid 418 snap -1 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: read oid 166 snap -1 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: seq_num 1268 ranges {0=413696} 2014-07-01 16:45:54,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: writing mira0435601-211 from 0 to 413696 tid 1 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 432 current snap is 0 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: seq_num 1269 ranges {3407872=282624} 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: writing mira0435601-432 from 3407872 to 3690496 tid 1 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1915: read oid 484 snap -1 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1911: done (4 left) 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 410 current snap is 0 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: seq_num 1270 ranges {1024000=155648} 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: writing mira0435601-410 from 1024000 to 1179648 tid 1 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 310 current snap is 0 2014-07-01 16:45:54,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: seq_num 1271 ranges {3579904=585728,4165632=503808,4669440=299008} 2014-07-01 16:45:54,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: writing mira0435601-310 from 3579904 to 4165632 tid 1 2014-07-01 16:45:54,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: writing mira0435601-310 from 4165632 to 4669440 tid 2 2014-07-01 16:45:54,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: writing mira0435601-310 from 4669440 to 4968448 tid 3 2014-07-01 16:45:54,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1918: read oid 80 snap -1 2014-07-01 16:45:54,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:1919: delete oid 315 current snap is 0 2014-07-01 16:45:55,031.031 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:55,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1789 16 7213 7197 16.0886 0 - 3.96258 2014-07-01 16:45:55,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:1918: expect (ObjNum 1052 snap 0 seq_num 1052) 2014-07-01 16:45:56,593.593 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:56,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1790 16 7214 7198 16.0819 0.8 4.33104 3.96263 2014-07-01 16:45:57,301.301 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'15 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'37 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2417/15713 objects degraded (15.382%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 1116754, u'degraded_objects': 2417, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 31233265451, u'degraded_total': 15713, u'bytes_used': 61116506112, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 8934032, u'op_per_sec': 3, u'version': 1433, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 207, u'state_name': u'active+clean'}, {u'count': 15, u'state_name': u'active+recovering'}], u'read_bytes_sec': 817, u'degraded_ratio': u'15.382', u'bytes_avail': 4937466032128}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:45:57,301.301 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:45:57,599.599 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1791 16 7214 7198 16.0729 0 - 3.96263 2014-07-01 16:45:57,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 1 to mira0435601-410 2014-07-01 16:45:57,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 3 to mira0435601-410 2014-07-01 16:45:57,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 4 to mira0435601-410 2014-07-01 16:45:57,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 296 (ObjNum 1270 snap 0 seq_num 1270) dirty exists 2014-07-01 16:45:58,074.074 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:45:58,074.074 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:45:58,600.600 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1792 16 7214 7198 16.0639 0 - 3.96263 2014-07-01 16:45:58,892.892 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:45:59,616.616 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1793 16 7214 7198 16.055 0 - 3.96263 2014-07-01 16:46:00,600.600 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1794 16 7214 7198 16.046 0 - 3.96263 2014-07-01 16:46:01,600.600 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1795 16 7214 7198 16.0371 0 - 3.96263 2014-07-01 16:46:02,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1796 16 7214 7198 16.0282 0 - 3.96263 2014-07-01 16:46:03,275.275 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:03,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1797 16 7214 7198 16.0192 0 - 3.96263 2014-07-01 16:46:03,863.863 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:04,610.610 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1798 16 7214 7198 16.0103 0 - 3.96263 2014-07-01 16:46:05,612.612 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:05,613.613 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1799 16 7214 7198 16.0014 0 - 3.96263 2014-07-01 16:46:06,349.349 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:45:11.320010', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886672000, u'health': u'HEALTH_OK', u'kb_used': 10109052, u'store_stats': {u'bytes_total': 8877043, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2373/15715 objects degraded (15.100%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 15619656, u'degraded_objects': 2373, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 31239550763, u'degraded_total': 15715, u'bytes_used': 61089632256, u'recovering_objects_per_sec': 3, u'version': 1436, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'15.100', u'bytes_avail': 4937492905984}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:06,349.349 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:06,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:2014-07-01 07:46:06.513784min lat: 0.057269 max lat: 294.525 avg lat: 3.96263 2014-07-01 16:46:06,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat avg lat 2014-07-01 16:46:06,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1800 16 7214 7198 15.9925 0 - 3.96263 2014-07-01 16:46:07,127.127 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:46:07,128.128 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:07,602.602 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1801 16 7214 7198 15.9837 0 - 3.96263 2014-07-01 16:46:07,940.940 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:08,601.601 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1802 16 7214 7198 15.9748 0 - 3.96263 2014-07-01 16:46:09,602.602 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1803 16 7214 7198 15.9659 0 - 3.96263 2014-07-01 16:46:10,603.603 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1804 16 7214 7198 15.9571 0 - 3.96263 2014-07-01 16:46:11,604.604 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1805 16 7214 7198 15.9482 0 - 3.96263 2014-07-01 16:46:12,465.465 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:12,602.602 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1806 16 7214 7198 15.9394 0 - 3.96263 2014-07-01 16:46:12,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 1 to mira0435601-211 2014-07-01 16:46:12,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: done (7 left) 2014-07-01 16:46:12,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1918: done (6 left) 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1919: done (5 left) 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1920: read oid 155 snap -1 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1921: read oid 101 snap -1 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1922: read oid 380 snap -1 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 227 current snap is 0 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: seq_num 1273 ranges {1880064=778240,2658304=667648,3325952=536576} 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: writing mira0435601-227 from 1880064 to 2658304 tid 1 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: writing mira0435601-227 from 2658304 to 3325952 tid 2 2014-07-01 16:46:12,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: writing mira0435601-227 from 3325952 to 3862528 tid 3 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: delete oid 399 current snap is 0 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 3 to mira0435601-211 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: finishing write tid 1 to mira0435601-432 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: finishing write tid 3 to mira0435601-432 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 4 to mira0435601-211 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 111 (ObjNum 1268 snap 0 seq_num 1268) dirty exists 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: finishing write tid 4 to mira0435601-432 2014-07-01 16:46:12,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 108 (ObjNum 1269 snap 0 seq_num 1269) dirty exists 2014-07-01 16:46:12,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: finishing write tid 1 to mira0435601-227 2014-07-01 16:46:12,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: finishing write tid 2 to mira0435601-227 2014-07-01 16:46:12,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: finishing write tid 3 to mira0435601-227 2014-07-01 16:46:12,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: finishing write tid 5 to mira0435601-227 2014-07-01 16:46:13,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: finishing write tid 6 to mira0435601-227 2014-07-01 16:46:13,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 181 (ObjNum 1273 snap 0 seq_num 1273) dirty exists 2014-07-01 16:46:13,066.066 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:13,968.968 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1807 16 7215 7199 15.9328 0.235294 29.7018 3.96621 2014-07-01 16:46:14,604.604 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1808 16 7215 7199 15.924 0 - 3.96621 2014-07-01 16:46:14,934.934 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:15,603.603 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1809 16 7215 7199 15.9152 0 - 3.96621 2014-07-01 16:46:15,637.637 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2377/15730 objects degraded (15.111%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2377, u'num_pgs': 244, u'data_bytes': 31265412907, u'degraded_total': 15730, u'bytes_used': 61169569792, u'op_per_sec': 1, u'write_bytes_sec': 3296878, u'version': 1438, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 261, u'degraded_ratio': u'15.111', u'bytes_avail': 4937412968448}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:15,637.637 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:16,404.404 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:46:16,405.405 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:16,603.603 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1810 16 7215 7199 15.9064 0 - 3.96621 2014-07-01 16:46:17,193.193 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:17,604.604 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1811 16 7215 7199 15.8976 0 - 3.96621 2014-07-01 16:46:19,456.456 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1812 16 7215 7199 15.8888 0 - 3.96621 2014-07-01 16:46:19,665.665 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1813 16 7215 7199 15.8801 0 - 3.96621 2014-07-01 16:46:20,604.604 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1814 16 7215 7199 15.8713 0 - 3.96621 2014-07-01 16:46:21,605.605 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1815 16 7215 7199 15.8626 0 - 3.96621 2014-07-01 16:46:22,605.605 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1816 16 7215 7199 15.8538 0 - 3.96621 2014-07-01 16:46:22,917.917 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:23,525.525 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:23,605.605 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 1817 16 7215 7199 15.8451 0 - 3.96621 2014-07-01 16:46:24,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1915: expect (ObjNum 1009 snap 0 seq_num 1009) 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: Total time run: 1818.299637 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Total writes made: 7215 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Write size: 4194304 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Bandwidth (MB/sec): 15.872 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout: 2014-07-01 16:46:24,606.606 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Stddev Bandwidth: 17.8782 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Max bandwidth (MB/sec): 120 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Min bandwidth (MB/sec): 0 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Average Latency: 4.03225 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Stddev Latency: 6.10015 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Max latency: 294.525 2014-07-01 16:46:24,607.607 INFO:teuthology.task.radosbench.radosbench.0.mira043.stdout:Min latency: 0.057269 2014-07-01 16:46:25,110.110 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:25,829.829 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2379/15732 objects degraded (15.122%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 2379, u'num_pgs': 244, u'data_bytes': 31282601771, u'degraded_total': 15732, u'bytes_used': 61105528832, u'op_per_sec': 0, u'write_bytes_sec': 4174369, u'version': 1441, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'15.122', u'bytes_avail': 4937477009408}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:25,829.829 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:26,598.598 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:46:26,598.598 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:27,439.439 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:29,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: finishing write tid 1 to mira0435601-310 2014-07-01 16:46:29,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: finishing write tid 2 to mira0435601-310 2014-07-01 16:46:29,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: finishing write tid 3 to mira0435601-310 2014-07-01 16:46:29,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: finishing write tid 5 to mira0435601-310 2014-07-01 16:46:29,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: expect (ObjNum 864 snap 0 seq_num 864) 2014-07-01 16:46:31,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:1920: expect (ObjNum 1176 snap 0 seq_num 1176) 2014-07-01 16:46:31,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:1922: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 16:46:31,967.967 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:32,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: finishing write tid 6 to mira0435601-310 2014-07-01 16:46:32,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 225 (ObjNum 1271 snap 0 seq_num 1271) dirty exists 2014-07-01 16:46:32,729.729 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:34,283.283 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:35,018.018 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'12 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2378/15712 objects degraded (15.135%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 5223931, u'degraded_objects': 2378, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 31237852983, u'degraded_total': 15712, u'bytes_used': 61082898432, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 346136, u'op_per_sec': 7, u'version': 1446, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1331745, u'degraded_ratio': u'15.135', u'bytes_avail': 4937499639808}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:35,018.018 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:35,786.786 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:46:35,786.786 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:36,604.604 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:40,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: done (9 left) 2014-07-01 16:46:40,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: done (8 left) 2014-07-01 16:46:40,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: done (7 left) 2014-07-01 16:46:40,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1915: done (6 left) 2014-07-01 16:46:40,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: done (5 left) 2014-07-01 16:46:40,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1920: done (4 left) 2014-07-01 16:46:40,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1921: done (3 left) 2014-07-01 16:46:40,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:1922: done (2 left) 2014-07-01 16:46:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: done (1 left) 2014-07-01 16:46:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: done (0 left) 2014-07-01 16:46:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: read oid 113 snap -1 2014-07-01 16:46:40,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:46:40,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: seq_num 1275 ranges {4554752=675840} 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: writing mira0435601-147 from 4554752 to 5230592 tid 1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: read oid 133 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: read oid 139 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: read oid 268 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: read oid 204 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: read oid 116 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1932: read oid 478 snap -1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 146 current snap is 0 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: seq_num 1276 ranges {0=598016,598016=663552,1261568=139264} 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: writing mira0435601-146 from 0 to 598016 tid 1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: writing mira0435601-146 from 598016 to 1261568 tid 2 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: writing mira0435601-146 from 1261568 to 1400832 tid 3 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: seq_num 1277 ranges {5017600=225280} 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: writing mira0435601-292 from 5017600 to 5242880 tid 1 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 271 current snap is 0 2014-07-01 16:46:40,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: seq_num 1278 ranges {81920=94208} 2014-07-01 16:46:40,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: writing mira0435601-271 from 81920 to 176128 tid 1 2014-07-01 16:46:40,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1936: read oid 102 snap -1 2014-07-01 16:46:40,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: expect (ObjNum 991 snap 0 seq_num 991) 2014-07-01 16:46:41,135.135 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:41,911.911 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:42,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: expect (ObjNum 900 snap 0 seq_num 900) 2014-07-01 16:46:43,476.476 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: finishing write tid 1 to mira0435601-292 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: finishing write tid 3 to mira0435601-292 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: done (11 left) 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: done (10 left) 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1937: delete oid 18 current snap is 0 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: finishing write tid 4 to mira0435601-292 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 243 (ObjNum 1277 snap 0 seq_num 1277) dirty exists 2014-07-01 16:46:43,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1936: expect (ObjNum 1253 snap 0 seq_num 1253) 2014-07-01 16:46:44,187.187 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:45:44.712104', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883029164, u'health': u'HEALTH_OK', u'kb_used': 13751888, u'store_stats': {u'bytes_total': 7828931, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'8 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2347/15612 objects degraded (15.033%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 659796, u'degraded_objects': 2347, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 31019777863, u'degraded_total': 15612, u'bytes_used': 60738478080, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 1137945, u'op_per_sec': 28, u'version': 1452, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 4766318, u'degraded_ratio': u'15.033', u'bytes_avail': 4937844060160}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:44,187.187 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:44,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: expect (ObjNum 1037 snap 0 seq_num 1037) 2014-07-01 16:46:44,967.967 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:46:44,967.967 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:45,764.764 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:46,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: finishing write tid 1 to mira0435601-147 2014-07-01 16:46:46,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: finishing write tid 3 to mira0435601-147 2014-07-01 16:46:46,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: expect (ObjNum 1250 snap 0 seq_num 1250) 2014-07-01 16:46:47,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: expect (ObjNum 930 snap 0 seq_num 930) 2014-07-01 16:46:49,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: finishing write tid 1 to mira0435601-271 2014-07-01 16:46:49,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: finishing write tid 3 to mira0435601-271 2014-07-01 16:46:49,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: finishing write tid 4 to mira0435601-147 2014-07-01 16:46:49,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 125 (ObjNum 1275 snap 0 seq_num 1275) dirty exists 2014-07-01 16:46:49,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: finishing write tid 4 to mira0435601-271 2014-07-01 16:46:49,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 271 v 181 (ObjNum 1278 snap 0 seq_num 1278) dirty exists 2014-07-01 16:46:49,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:46:50,455.455 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:51,209.209 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:51,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: finishing write tid 1 to mira0435601-146 2014-07-01 16:46:51,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: finishing write tid 2 to mira0435601-146 2014-07-01 16:46:51,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: finishing write tid 3 to mira0435601-146 2014-07-01 16:46:51,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: finishing write tid 5 to mira0435601-146 2014-07-01 16:46:51,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1932: expect (ObjNum 477 snap 0 seq_num 477) 2014-07-01 16:46:52,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: finishing write tid 6 to mira0435601-146 2014-07-01 16:46:52,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 135 (ObjNum 1276 snap 0 seq_num 1276) dirty exists 2014-07-01 16:46:52,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: done (10 left) 2014-07-01 16:46:52,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: done (9 left) 2014-07-01 16:46:52,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: done (8 left) 2014-07-01 16:46:52,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: done (7 left) 2014-07-01 16:46:52,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: done (6 left) 2014-07-01 16:46:52,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1932: done (5 left) 2014-07-01 16:46:52,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: done (4 left) 2014-07-01 16:46:52,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: done (3 left) 2014-07-01 16:46:52,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: done (2 left) 2014-07-01 16:46:52,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1936: done (1 left) 2014-07-01 16:46:52,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1937: done (0 left) 2014-07-01 16:46:52,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1938: read oid 36 snap -1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1939: read oid 401 snap -1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 456 current snap is 0 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: seq_num 1280 ranges {1703936=798720,2502656=401408} 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: writing mira0435601-456 from 1703936 to 2502656 tid 1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: writing mira0435601-456 from 2502656 to 2904064 tid 2 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 353 current snap is 0 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: seq_num 1281 ranges {626688=716800,1343488=475136,1818624=581632,2400256=147456} 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: writing mira0435601-353 from 626688 to 1343488 tid 1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: writing mira0435601-353 from 1343488 to 1818624 tid 2 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: writing mira0435601-353 from 1818624 to 2400256 tid 3 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: writing mira0435601-353 from 2400256 to 2547712 tid 4 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 254 current snap is 0 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: seq_num 1282 ranges {1343488=745472} 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: writing mira0435601-254 from 1343488 to 2088960 tid 1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: read oid 241 snap -1 2014-07-01 16:46:52,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:1938: expect (ObjNum 35 snap 0 seq_num 35) 2014-07-01 16:46:52,797.797 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:53,502.502 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:45:54.537409', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886665384, u'health': u'HEALTH_OK', u'kb_used': 10115668, u'store_stats': {u'bytes_total': 9925577, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'36 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'14 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2286/15411 objects degraded (14.834%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 20858954, u'degraded_objects': 2286, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 30608199494, u'degraded_total': 15411, u'bytes_used': 60064014336, u'recovering_objects_per_sec': 4, u'write_bytes_sec': 597352, u'op_per_sec': 32, u'version': 1459, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 208, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 5601, u'degraded_ratio': u'14.834', u'bytes_avail': 4938518523904}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:46:53,503.503 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:46:53,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 1 to mira0435601-353 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 2 to mira0435601-353 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: finishing write tid 1 to mira0435601-456 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: finishing write tid 2 to mira0435601-456 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 3 to mira0435601-353 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: finishing write tid 4 to mira0435601-456 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 4 to mira0435601-353 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1938: done (5 left) 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1939: done (4 left) 2014-07-01 16:46:53,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 348 current snap is 0 2014-07-01 16:46:53,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: seq_num 1283 ranges {901120=614400} 2014-07-01 16:46:53,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: writing mira0435601-348 from 901120 to 1515520 tid 1 2014-07-01 16:46:53,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: seq_num 1284 ranges {2056192=487424} 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: writing mira0435601-11 from 2056192 to 2543616 tid 1 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 366 current snap is 0 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: seq_num 1285 ranges {2293760=163840} 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: writing mira0435601-366 from 2293760 to 2457600 tid 1 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: seq_num 1286 ranges {4689920=32768} 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: writing mira0435601-238 from 4689920 to 4722688 tid 1 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: read oid 38 snap -1 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 6 to mira0435601-353 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 1 to mira0435601-254 2014-07-01 16:46:53,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 446 current snap is 0 2014-07-01 16:46:53,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: seq_num 1287 ranges {5083136=708608,5791744=671744,6463488=528384,6991872=405504} 2014-07-01 16:46:53,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: writing mira0435601-446 from 5083136 to 5791744 tid 1 2014-07-01 16:46:53,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: writing mira0435601-446 from 5791744 to 6463488 tid 2 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: writing mira0435601-446 from 6463488 to 6991872 tid 3 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: writing mira0435601-446 from 6991872 to 7397376 tid 4 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 171 current snap is 0 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: seq_num 1288 ranges {2596864=737280,3334144=401408} 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: writing mira0435601-171 from 2596864 to 3334144 tid 1 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: writing mira0435601-171 from 3334144 to 3735552 tid 2 2014-07-01 16:46:54,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: read oid 198 snap -1 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 1 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 2 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 3 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 4 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 6 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 7 to mira0435601-446 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 238 (ObjNum 1287 snap 0 seq_num 1287) dirty exists 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: done (11 left) 2014-07-01 16:46:54,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: seq_num 1289 ranges {5537792=606208,6144000=401408} 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: writing mira0435601-359 from 5537792 to 6144000 tid 1 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: writing mira0435601-359 from 6144000 to 6545408 tid 2 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: read oid 431 snap -1 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1954: read oid 123 snap -1 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 332 current snap is 0 2014-07-01 16:46:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: seq_num 1290 ranges {950272=528384,1478656=667648,2146304=536576} 2014-07-01 16:46:54,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: writing mira0435601-332 from 950272 to 1478656 tid 1 2014-07-01 16:46:54,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: writing mira0435601-332 from 1478656 to 2146304 tid 2 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: writing mira0435601-332 from 2146304 to 2682880 tid 3 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 134 current snap is 0 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: seq_num 1291 ranges {0=557056} 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: writing mira0435601-134 from 0 to 557056 tid 1 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: finishing write tid 1 to mira0435601-134 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: finishing write tid 3 to mira0435601-134 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: finishing write tid 4 to mira0435601-134 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 311 (ObjNum 1291 snap 0 seq_num 1291) dirty exists 2014-07-01 16:46:54,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: expect (ObjNum 1130 snap 0 seq_num 1130) 2014-07-01 16:46:54,290.290 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:46:54,290.290 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:46:55,098.098 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:46:55,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 3 to mira0435601-254 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 1 to mira0435601-348 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: finishing write tid 5 to mira0435601-456 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 179 (ObjNum 1280 snap 0 seq_num 1280) dirty exists 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 3 to mira0435601-348 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 1 to mira0435601-238 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 3 to mira0435601-238 2014-07-01 16:46:55,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: expect (ObjNum 1094 snap 0 seq_num 1094) 2014-07-01 16:46:56,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: finishing write tid 1 to mira0435601-366 2014-07-01 16:46:56,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: finishing write tid 1 to mira0435601-171 2014-07-01 16:46:56,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: expect (ObjNum 1118 snap 0 seq_num 1118) 2014-07-01 16:46:58,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: finishing write tid 2 to mira0435601-171 2014-07-01 16:46:58,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: finishing write tid 1 to mira0435601-11 2014-07-01 16:46:58,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: finishing write tid 3 to mira0435601-11 2014-07-01 16:46:58,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 7 to mira0435601-353 2014-07-01 16:46:58,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 183 (ObjNum 1281 snap 0 seq_num 1281) dirty exists 2014-07-01 16:46:58,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: finishing write tid 4 to mira0435601-171 2014-07-01 16:46:58,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: finishing write tid 4 to mira0435601-11 2014-07-01 16:46:58,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 114 (ObjNum 1284 snap 0 seq_num 1284) dirty exists 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 4 to mira0435601-254 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 183 (ObjNum 1282 snap 0 seq_num 1282) dirty exists 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 4 to mira0435601-348 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 223 (ObjNum 1283 snap 0 seq_num 1283) dirty exists 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: finishing write tid 3 to mira0435601-366 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 4 to mira0435601-238 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 176 (ObjNum 1286 snap 0 seq_num 1286) dirty exists 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: finishing write tid 4 to mira0435601-366 2014-07-01 16:46:58,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 197 (ObjNum 1285 snap 0 seq_num 1285) dirty exists 2014-07-01 16:46:58,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: finishing write tid 5 to mira0435601-171 2014-07-01 16:46:58,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 226 (ObjNum 1288 snap 0 seq_num 1288) dirty exists 2014-07-01 16:46:58,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: expect (ObjNum 1189 snap 0 seq_num 1189) 2014-07-01 16:46:59,628.628 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: finishing write tid 1 to mira0435601-359 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: finishing write tid 2 to mira0435601-359 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: finishing write tid 4 to mira0435601-359 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: finishing write tid 5 to mira0435601-359 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 198 (ObjNum 1289 snap 0 seq_num 1289) dirty exists 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: done (15 left) 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: done (14 left) 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: done (13 left) 2014-07-01 16:47:00,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: done (12 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: done (11 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: done (10 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: done (9 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: done (8 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: done (7 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: done (6 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: done (5 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: done (4 left) 2014-07-01 16:47:00,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: done (3 left) 2014-07-01 16:47:00,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: done (2 left) 2014-07-01 16:47:00,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1957: read oid 212 snap -1 2014-07-01 16:47:00,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1954: expect (ObjNum 931 snap 0 seq_num 931) 2014-07-01 16:47:00,400.400 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: finishing write tid 1 to mira0435601-332 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: finishing write tid 2 to mira0435601-332 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: finishing write tid 3 to mira0435601-332 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: finishing write tid 5 to mira0435601-332 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: finishing write tid 6 to mira0435601-332 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 123 (ObjNum 1290 snap 0 seq_num 1290) dirty exists 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1954: done (2 left) 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: done (1 left) 2014-07-01 16:47:00,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:1958: delete oid 339 current snap is 0 2014-07-01 16:47:01,970.970 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:02,681.681 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:46:11.320389', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886663600, u'health': u'HEALTH_OK', u'kb_used': 10117452, u'store_stats': {u'bytes_total': 9925619, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'15 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2190/15078 objects degraded (14.524%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 12620327, u'degraded_objects': 2190, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29905882950, u'degraded_total': 15078, u'bytes_used': 58660360192, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 0, u'op_per_sec': 19, u'version': 1466, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 209, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1369, u'degraded_ratio': u'14.524', u'bytes_avail': 4939922178048}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:02,681.681 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:03,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1957: expect (ObjNum 976 snap 0 seq_num 976) 2014-07-01 16:47:03,465.465 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:47:03,465.465 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:04,280.280 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:04,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1957: done (1 left) 2014-07-01 16:47:04,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1958: done (0 left) 2014-07-01 16:47:04,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: read oid 487 snap -1 2014-07-01 16:47:04,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:47:04,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: seq_num 1293 ranges {3846144=299008} 2014-07-01 16:47:04,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: writing mira0435601-27 from 3846144 to 4145152 tid 1 2014-07-01 16:47:04,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1961: delete oid 197 current snap is 0 2014-07-01 16:47:05,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1961: done (2 left) 2014-07-01 16:47:05,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 16:47:05,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: seq_num 1295 ranges {4722688=802816,5525504=401408} 2014-07-01 16:47:05,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: writing mira0435601-238 from 4722688 to 5525504 tid 1 2014-07-01 16:47:05,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: writing mira0435601-238 from 5525504 to 5926912 tid 2 2014-07-01 16:47:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 388 current snap is 0 2014-07-01 16:47:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: seq_num 1296 ranges {1036288=520192,1556480=659456,2215936=696320,2912256=249856} 2014-07-01 16:47:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: writing mira0435601-388 from 1036288 to 1556480 tid 1 2014-07-01 16:47:05,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: writing mira0435601-388 from 1556480 to 2215936 tid 2 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: writing mira0435601-388 from 2215936 to 2912256 tid 3 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: writing mira0435601-388 from 2912256 to 3162112 tid 4 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 369 current snap is 0 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: seq_num 1297 ranges {0=548864,548864=401408} 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: writing mira0435601-369 from 0 to 548864 tid 1 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: writing mira0435601-369 from 548864 to 950272 tid 2 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: read oid 263 snap -1 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: finishing write tid 1 to mira0435601-27 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: finishing write tid 3 to mira0435601-27 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 325 current snap is 0 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: seq_num 1298 ranges {1593344=573440,2166784=471040,2637824=643072,3280896=491520} 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: writing mira0435601-325 from 1593344 to 2166784 tid 1 2014-07-01 16:47:05,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: writing mira0435601-325 from 2166784 to 2637824 tid 2 2014-07-01 16:47:05,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: writing mira0435601-325 from 2637824 to 3280896 tid 3 2014-07-01 16:47:05,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: writing mira0435601-325 from 3280896 to 3772416 tid 4 2014-07-01 16:47:05,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 195 current snap is 0 2014-07-01 16:47:05,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: seq_num 1299 ranges {1273856=741376,2015232=458752,2473984=344064} 2014-07-01 16:47:05,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: writing mira0435601-195 from 1273856 to 2015232 tid 1 2014-07-01 16:47:05,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: writing mira0435601-195 from 2015232 to 2473984 tid 2 2014-07-01 16:47:05,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: writing mira0435601-195 from 2473984 to 2818048 tid 3 2014-07-01 16:47:05,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1968: read oid 462 snap -1 2014-07-01 16:47:05,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: expect (ObjNum 486 snap 0 seq_num 486) 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: finishing write tid 4 to mira0435601-27 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 199 (ObjNum 1293 snap 0 seq_num 1293) dirty exists 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: finishing write tid 1 to mira0435601-238 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 1 to mira0435601-388 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: finishing write tid 2 to mira0435601-238 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 2 to mira0435601-388 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: finishing write tid 4 to mira0435601-238 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 3 to mira0435601-388 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: finishing write tid 5 to mira0435601-238 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 179 (ObjNum 1295 snap 0 seq_num 1295) dirty exists 2014-07-01 16:47:06,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: expect (ObjNum 897 snap 0 seq_num 897) 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: finishing write tid 1 to mira0435601-369 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: finishing write tid 2 to mira0435601-369 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: finishing write tid 4 to mira0435601-369 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: finishing write tid 1 to mira0435601-195 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: finishing write tid 2 to mira0435601-195 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: finishing write tid 3 to mira0435601-195 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: finishing write tid 5 to mira0435601-195 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: finishing write tid 5 to mira0435601-369 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 369 v 241 (ObjNum 1297 snap 0 seq_num 1297) dirty exists 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: finishing write tid 6 to mira0435601-195 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 224 (ObjNum 1299 snap 0 seq_num 1299) dirty exists 2014-07-01 16:47:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: done (8 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: done (7 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: done (6 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: done (5 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: done (4 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: done (3 left) 2014-07-01 16:47:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1969: read oid 85 snap -1 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 304 current snap is 0 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: seq_num 1300 ranges {0=368640} 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: writing mira0435601-304 from 0 to 368640 tid 1 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 332 current snap is 0 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: seq_num 1301 ranges {2682880=794624,3477504=622592,4100096=180224} 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: writing mira0435601-332 from 2682880 to 3477504 tid 1 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: writing mira0435601-332 from 3477504 to 4100096 tid 2 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: writing mira0435601-332 from 4100096 to 4280320 tid 3 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: read oid 193 snap -1 2014-07-01 16:47:07,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1969: expect (ObjNum 84 snap 0 seq_num 84) 2014-07-01 16:47:08,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 4 to mira0435601-388 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: finishing write tid 1 to mira0435601-304 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: finishing write tid 3 to mira0435601-304 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: finishing write tid 4 to mira0435601-304 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 304 v 204 (ObjNum 1300 snap 0 seq_num 1300) dirty exists 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 6 to mira0435601-388 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: finishing write tid 7 to mira0435601-388 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 203 (ObjNum 1296 snap 0 seq_num 1296) dirty exists 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 1 to mira0435601-325 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 2 to mira0435601-325 2014-07-01 16:47:08,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1968: expect (ObjNum 461 snap 0 seq_num 461) 2014-07-01 16:47:09,009.009 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:09,850.850 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:10,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 3 to mira0435601-325 2014-07-01 16:47:10,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 4 to mira0435601-325 2014-07-01 16:47:10,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: done (6 left) 2014-07-01 16:47:10,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1968: done (5 left) 2014-07-01 16:47:10,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1969: done (4 left) 2014-07-01 16:47:10,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: done (3 left) 2014-07-01 16:47:10,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 298 current snap is 0 2014-07-01 16:47:10,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: seq_num 1302 ranges {3936256=110592} 2014-07-01 16:47:10,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: writing mira0435601-298 from 3936256 to 4046848 tid 1 2014-07-01 16:47:10,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 324 current snap is 0 2014-07-01 16:47:10,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: seq_num 1303 ranges {0=589824,589824=684032,1273856=765952,2039808=155648} 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: writing mira0435601-324 from 0 to 589824 tid 1 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: writing mira0435601-324 from 589824 to 1273856 tid 2 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: writing mira0435601-324 from 1273856 to 2039808 tid 3 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: writing mira0435601-324 from 2039808 to 2195456 tid 4 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 57 current snap is 0 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: seq_num 1304 ranges {1515520=614400,2129920=401408} 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: writing mira0435601-57 from 1515520 to 2129920 tid 1 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: writing mira0435601-57 from 2129920 to 2531328 tid 2 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1976: delete oid 17 current snap is 0 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 6 to mira0435601-325 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: finishing write tid 7 to mira0435601-325 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 142 (ObjNum 1298 snap 0 seq_num 1298) dirty exists 2014-07-01 16:47:10,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: expect (ObjNum 1060 snap 0 seq_num 1060) 2014-07-01 16:47:11,543.543 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: finishing write tid 1 to mira0435601-332 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: finishing write tid 2 to mira0435601-332 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: finishing write tid 3 to mira0435601-332 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: finishing write tid 5 to mira0435601-332 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: finishing write tid 6 to mira0435601-332 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 127 (ObjNum 1301 snap 0 seq_num 1301) dirty exists 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 1 to mira0435601-324 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 2 to mira0435601-324 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 3 to mira0435601-324 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 4 to mira0435601-324 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 6 to mira0435601-324 2014-07-01 16:47:12,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: finishing write tid 7 to mira0435601-324 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 209 (ObjNum 1303 snap 0 seq_num 1303) dirty exists 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: finishing write tid 1 to mira0435601-298 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: finishing write tid 3 to mira0435601-298 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 1 to mira0435601-57 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: finishing write tid 4 to mira0435601-298 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 129 (ObjNum 1302 snap 0 seq_num 1302) dirty exists 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 2 to mira0435601-57 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 4 to mira0435601-57 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 5 to mira0435601-57 2014-07-01 16:47:12,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 128 (ObjNum 1304 snap 0 seq_num 1304) dirty exists 2014-07-01 16:47:12,278.278 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2114/14737 objects degraded (14.345%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 2780041, u'degraded_objects': 2114, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 29199339334, u'degraded_total': 14737, u'bytes_used': 57230860288, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 886409, u'op_per_sec': 31, u'version': 1473, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 209, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2870989, u'degraded_ratio': u'14.345', u'bytes_avail': 4941351677952}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:12,278.278 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:13,050.050 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:47:13,050.050 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:13,865.865 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:16,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: done (6 left) 2014-07-01 16:47:16,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: done (5 left) 2014-07-01 16:47:16,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: done (4 left) 2014-07-01 16:47:16,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: done (3 left) 2014-07-01 16:47:16,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: done (2 left) 2014-07-01 16:47:16,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: done (1 left) 2014-07-01 16:47:16,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:1976: done (0 left) 2014-07-01 16:47:16,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: read oid 258 snap -1 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1978: delete oid 223 current snap is 0 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1978: done (1 left) 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: seq_num 1307 ranges {4640768=786432,5427200=663552,6090752=540672} 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: writing mira0435601-252 from 4640768 to 5427200 tid 1 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: writing mira0435601-252 from 5427200 to 6090752 tid 2 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: writing mira0435601-252 from 6090752 to 6631424 tid 3 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: seq_num 1308 ranges {4657152=307200} 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: writing mira0435601-9 from 4657152 to 4964352 tid 1 2014-07-01 16:47:16,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 256 current snap is 0 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: seq_num 1309 ranges {1085440=176128} 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: writing mira0435601-256 from 1085440 to 1261568 tid 1 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 144 current snap is 0 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: seq_num 1310 ranges {0=49152} 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: writing mira0435601-144 from 0 to 49152 tid 1 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1983: read oid 122 snap -1 2014-07-01 16:47:16,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: expect (ObjNum 1042 snap 0 seq_num 1042) 2014-07-01 16:47:18,250.250 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: finishing write tid 1 to mira0435601-252 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: finishing write tid 2 to mira0435601-252 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: finishing write tid 3 to mira0435601-252 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: finishing write tid 5 to mira0435601-252 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: finishing write tid 6 to mira0435601-252 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 300 (ObjNum 1307 snap 0 seq_num 1307) dirty exists 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 1 to mira0435601-9 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 3 to mira0435601-9 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 4 to mira0435601-9 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 313 (ObjNum 1308 snap 0 seq_num 1308) dirty exists 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: finishing write tid 1 to mira0435601-144 2014-07-01 16:47:18,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: finishing write tid 3 to mira0435601-144 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: finishing write tid 1 to mira0435601-256 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: finishing write tid 3 to mira0435601-256 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: finishing write tid 4 to mira0435601-256 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 256 v 144 (ObjNum 1309 snap 0 seq_num 1309) dirty exists 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: finishing write tid 4 to mira0435601-144 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 101 (ObjNum 1310 snap 0 seq_num 1310) dirty exists 2014-07-01 16:47:18,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: done (5 left) 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: done (4 left) 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: done (3 left) 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: done (2 left) 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: done (1 left) 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1984: read oid 54 snap -1 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: seq_num 1311 ranges {413696=479232,892928=643072,1536000=159744} 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: writing mira0435601-211 from 413696 to 892928 tid 1 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: writing mira0435601-211 from 892928 to 1536000 tid 2 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: writing mira0435601-211 from 1536000 to 1695744 tid 3 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 396 current snap is 0 2014-07-01 16:47:18,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: seq_num 1312 ranges {1056768=798720,1855488=425984,2281472=483328,2764800=294912} 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: writing mira0435601-396 from 1056768 to 1855488 tid 1 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: writing mira0435601-396 from 1855488 to 2281472 tid 2 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: writing mira0435601-396 from 2281472 to 2764800 tid 3 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: writing mira0435601-396 from 2764800 to 3059712 tid 4 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: delete oid 341 current snap is 0 2014-07-01 16:47:18,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:1983: expect (ObjNum 896 snap 0 seq_num 896) 2014-07-01 16:47:18,853.853 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:20,600.600 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:21,305.305 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 2022/14480 objects degraded (13.964%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 1540673, u'degraded_objects': 2022, u'num_pgs': 244, u'recovering_keys_per_sec': 0, u'data_bytes': 28671164231, u'degraded_total': 14480, u'bytes_used': 56386461696, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 844060, u'op_per_sec': 25, u'version': 1479, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 209, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'read_bytes_sec': 8275, u'degraded_ratio': u'13.964', u'bytes_avail': 4942196076544}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 600, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:21,306.306 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:22,086.086 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:47:22,086.086 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:22,898.898 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:23,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1984: expect (ObjNum 723 snap 0 seq_num 723) 2014-07-01 16:47:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: finishing write tid 1 to mira0435601-211 2014-07-01 16:47:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: finishing write tid 2 to mira0435601-211 2014-07-01 16:47:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: finishing write tid 3 to mira0435601-211 2014-07-01 16:47:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: finishing write tid 5 to mira0435601-211 2014-07-01 16:47:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 1 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 2 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 3 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: finishing write tid 6 to mira0435601-211 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 115 (ObjNum 1311 snap 0 seq_num 1311) dirty exists 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 4 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 6 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: finishing write tid 7 to mira0435601-396 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 250 (ObjNum 1312 snap 0 seq_num 1312) dirty exists 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1983: done (4 left) 2014-07-01 16:47:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1984: done (3 left) 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: done (2 left) 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: done (1 left) 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: done (0 left) 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 308 current snap is 0 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: seq_num 1314 ranges {1568768=700416} 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: writing mira0435601-308 from 1568768 to 2269184 tid 1 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1989: read oid 460 snap -1 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 487 current snap is 0 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: seq_num 1315 ranges {1146880=569344} 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: writing mira0435601-487 from 1146880 to 1716224 tid 1 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1991: read oid 483 snap -1 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 218 current snap is 0 2014-07-01 16:47:24,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: seq_num 1316 ranges {0=442368} 2014-07-01 16:47:24,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: writing mira0435601-218 from 0 to 442368 tid 1 2014-07-01 16:47:24,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: read oid 223 snap -1 2014-07-01 16:47:24,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1994: delete oid 279 current snap is 0 2014-07-01 16:47:25,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: done (6 left) 2014-07-01 16:47:25,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:1994: done (5 left) 2014-07-01 16:47:25,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: read oid 364 snap -1 2014-07-01 16:47:25,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:1996: read oid 427 snap -1 2014-07-01 16:47:25,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: read oid 96 snap -1 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 362 current snap is 0 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: seq_num 1318 ranges {2547712=544768,3092480=466944,3559424=335872} 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: writing mira0435601-362 from 2547712 to 3092480 tid 1 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: writing mira0435601-362 from 3092480 to 3559424 tid 2 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: writing mira0435601-362 from 3559424 to 3895296 tid 3 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1999: read oid 356 snap -1 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 24 current snap is 0 2014-07-01 16:47:25,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: seq_num 1319 ranges {0=663552,663552=450560,1114112=548864,1662976=606208} 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: writing mira0435601-24 from 0 to 663552 tid 1 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: writing mira0435601-24 from 663552 to 1114112 tid 2 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: writing mira0435601-24 from 1114112 to 1662976 tid 3 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: writing mira0435601-24 from 1662976 to 2269184 tid 4 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 125 current snap is 0 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: seq_num 1320 ranges {491520=692224,1183744=401408} 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: writing mira0435601-125 from 491520 to 1183744 tid 1 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: writing mira0435601-125 from 1183744 to 1585152 tid 2 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: read oid 217 snap -1 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: finishing write tid 1 to mira0435601-362 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: finishing write tid 2 to mira0435601-362 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: finishing write tid 3 to mira0435601-362 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: finishing write tid 5 to mira0435601-362 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: finishing write tid 6 to mira0435601-362 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 245 (ObjNum 1318 snap 0 seq_num 1318) dirty exists 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: done (12 left) 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 307 current snap is 0 2014-07-01 16:47:25,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: seq_num 1321 ranges {0=765952} 2014-07-01 16:47:25,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: writing mira0435601-307 from 0 to 765952 tid 1 2014-07-01 16:47:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2004: read oid 140 snap -1 2014-07-01 16:47:25,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: finishing write tid 1 to mira0435601-125 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: finishing write tid 2 to mira0435601-125 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: finishing write tid 4 to mira0435601-125 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: finishing write tid 5 to mira0435601-125 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 227 (ObjNum 1320 snap 0 seq_num 1320) dirty exists 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: done (13 left) 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2005: read oid 296 snap -1 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 411 current snap is 0 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: seq_num 1322 ranges {2314240=483328,2797568=446464,3244032=757760} 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: writing mira0435601-411 from 2314240 to 2797568 tid 1 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: writing mira0435601-411 from 2797568 to 3244032 tid 2 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: writing mira0435601-411 from 3244032 to 4001792 tid 3 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: seq_num 1323 ranges {1953792=512000} 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: writing mira0435601-327 from 1953792 to 2465792 tid 1 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:47:25,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:1989: expect (ObjNum 876 snap 0 seq_num 876) 2014-07-01 16:47:26,867.867 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_create_defaults ... ok 2014-07-01 16:47:27,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 1 to mira0435601-308 2014-07-01 16:47:27,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1991: expect (ObjNum 482 snap 0 seq_num 482) 2014-07-01 16:47:27,423.423 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:28,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1999: expect (ObjNum 953 snap 0 seq_num 953) 2014-07-01 16:47:28,183.183 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:28,561.561 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_context_manager ... ok 2014-07-01 16:47:29,754.754 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:30,459.459 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'35 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1936/14217 objects degraded (13.618%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 1936, u'num_pgs': 244, u'data_bytes': 28121198402, u'degraded_total': 14217, u'bytes_used': 55189544960, u'version': 1488, u'pgs_by_state': [{u'count': 21, u'state_name': u'active+recovery_wait'}, {u'count': 209, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'13.618', u'bytes_avail': 4943392993280}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 602, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:30,459.459 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:30,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 1 to mira0435601-218 2014-07-01 16:47:30,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 3 to mira0435601-218 2014-07-01 16:47:30,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 3 to mira0435601-308 2014-07-01 16:47:30,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2004: expect (ObjNum 1076 snap 0 seq_num 1076) 2014-07-01 16:47:31,228.228 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:47:31,228.228 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:31,865.865 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_open_read_only ... ok 2014-07-01 16:47:31,971.971 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_remove_dne ... ok 2014-07-01 16:47:32,031.031 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_list_empty ... ok 2014-07-01 16:47:32,052.052 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:32,266.266 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_list ... ok 2014-07-01 16:47:34,013.013 INFO:teuthology.task.workunit.client.0.mira043.stderr:test_rbd.test_rename ... ok 2014-07-01 16:47:34,017.017 INFO:teuthology.task.workunit.client.0.mira043.stderr: 2014-07-01 16:47:34,019.019 INFO:teuthology.task.workunit.client.0.mira043.stderr:---------------------------------------------------------------------- 2014-07-01 16:47:34,019.019 INFO:teuthology.task.workunit.client.0.mira043.stderr:Ran 52 tests in 1670.791s 2014-07-01 16:47:34,020.020 INFO:teuthology.task.workunit.client.0.mira043.stderr: 2014-07-01 16:47:34,020.020 INFO:teuthology.task.workunit.client.0.mira043.stderr:OK (SKIP=14) 2014-07-01 16:47:34,179.179 INFO:teuthology.task.workunit.client.0.mira043.stderr:+ exit 0 2014-07-01 16:47:34,179.179 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp' 2014-07-01 16:47:34,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2005: expect (ObjNum 675 snap 0 seq_num 675) 2014-07-01 16:47:34,690.690 INFO:teuthology.task.workunit:Stopping rbd/test_librbd_python.sh on client.0... 2014-07-01 16:47:34,690.690 INFO:teuthology.orchestra.run.mira043:Running: 'rm -rf -- /home/ubuntu/cephtest/workunits.list /home/ubuntu/cephtest/workunit.client.0' 2014-07-01 16:47:35,237.237 DEBUG:teuthology.parallel:result is None 2014-07-01 16:47:35,237.237 INFO:teuthology.orchestra.run.mira043:Running: 'rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0' 2014-07-01 16:47:35,774.774 INFO:teuthology.task.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2014-07-01 16:47:35,774.774 INFO:teuthology.orchestra.run.mira043:Running: 'rmdir -- /home/ubuntu/cephtest/mnt.0' 2014-07-01 16:47:35,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 1 to mira0435601-487 2014-07-01 16:47:35,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 3 to mira0435601-487 2014-07-01 16:47:35,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 1 to mira0435601-24 2014-07-01 16:47:35,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: expect (ObjNum 363 snap 0 seq_num 363) 2014-07-01 16:47:36,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 2 to mira0435601-24 2014-07-01 16:47:36,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 3 to mira0435601-24 2014-07-01 16:47:36,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: expect (ObjNum 940 snap 0 seq_num 940) 2014-07-01 16:47:36,280.280 INFO:teuthology.task.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0 2014-07-01 16:47:36,280.280 INFO:teuthology.run_tasks:Running task rgw... 2014-07-01 16:47:36,281.281 INFO:teuthology.task.rgw:Using apache as radosgw frontend 2014-07-01 16:47:36,281.281 INFO:teuthology.task.rgw:Creating apache directories... 2014-07-01 16:47:36,281.281 INFO:teuthology.orchestra.run.mira043:Running: 'mkdir -p /home/ubuntu/cephtest/apache/htdocs.client.0 /home/ubuntu/cephtest/apache/tmp.client.0/fastcgi_sock && mkdir /home/ubuntu/cephtest/archive/apache.client.0' 2014-07-01 16:47:36,441.441 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:36,822.822 DEBUG:teuthology.task.rgw:In rgw.configure_regions_and_zones() and regions is None. Bailing 2014-07-01 16:47:36,822.822 INFO:teuthology.task.rgw:Configuring users... 2014-07-01 16:47:36,822.822 INFO:teuthology.task.rgw:creating data pools 2014-07-01 16:47:36,822.822 INFO:teuthology.orchestra.run.mira043:Running: 'ceph osd pool create .rgw.buckets 64 64' 2014-07-01 16:47:37,211.211 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:38,332.332 INFO:teuthology.orchestra.run.mira043.stderr:pool '.rgw.buckets' created 2014-07-01 16:47:38,343.343 INFO:teuthology.task.rgw:Shipping apache config and rgw.fcgi... 2014-07-01 16:47:38,343.343 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 16:47:38,793.793 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:38,884.884 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:47:38,885.885 INFO:teuthology.orchestra.run.mira043:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/apache/apache.client.0.conf' 2014-07-01 16:47:39,500.500 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:46:44.712496', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883015076, u'health': u'HEALTH_OK', u'kb_used': 13765976, u'store_stats': {u'bytes_total': 8877507, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634611}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'64 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'98 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1824/13863 objects degraded (13.157%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 66181985, u'degraded_objects': 1824, u'num_pgs': 308, u'recovering_keys_per_sec': 0, u'data_bytes': 27383512899, u'degraded_total': 13863, u'bytes_used': 53833261056, u'recovering_objects_per_sec': 15, u'write_bytes_sec': 0, u'op_per_sec': 34, u'version': 1496, u'pgs_by_state': [{u'count': 51, u'state_name': u'creating'}, {u'count': 13, u'state_name': u'peering'}, {u'count': 20, u'state_name': u'active+recovery_wait'}, {u'count': 210, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'13.157', u'bytes_avail': 4944749277184}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 604, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:39,501.501 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:39,745.745 INFO:teuthology.orchestra.run.mira043:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/apache/htdocs.client.0/rgw.fcgi' 2014-07-01 16:47:40,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 4 to mira0435601-24 2014-07-01 16:47:40,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 6 to mira0435601-24 2014-07-01 16:47:40,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: finishing write tid 7 to mira0435601-24 2014-07-01 16:47:40,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 24 v 120 (ObjNum 1319 snap 0 seq_num 1319) dirty exists 2014-07-01 16:47:40,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: expect (ObjNum 985 snap 0 seq_num 985) 2014-07-01 16:47:40,273.273 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:47:40,273.273 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:40,666.666 INFO:teuthology.orchestra.run.mira043:Running: 'chmod a=rx /home/ubuntu/cephtest/apache/htdocs.client.0/rgw.fcgi' 2014-07-01 16:47:41,097.097 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:41,244.244 INFO:teuthology.task.rgw:Starting rgw... 2014-07-01 16:47:41,245.245 INFO:teuthology.task.rgw:rgw client.0 config is {'idle_timeout': 300} 2014-07-01 16:47:41,245.245 INFO:teuthology.task.rgw:client client.0 is id 0 2014-07-01 16:47:41,245.245 INFO:teuthology.task.rgw.client.0:Restarting daemon 2014-07-01 16:47:41,246.246 INFO:teuthology.orchestra.run.mira043:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-socket-path /home/ubuntu/cephtest/apache/tmp.client.0/fastcgi_sock/rgw_sock -n client.0 -k /etc/ceph/ceph.client.0.keyring --log-file /var/log/ceph/rgw.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.client.0.stdout 2>&1' 2014-07-01 16:47:41,830.830 INFO:teuthology.task.rgw.client.0:Started 2014-07-01 16:47:41,830.830 INFO:teuthology.task.rgw:Starting apache... 2014-07-01 16:47:41,830.830 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 16:47:42,205.205 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 16:47:42,206.206 INFO:teuthology.orchestra.run.mira043:Running: 'adjust-ulimits daemon-helper kill apache2 -X -f /home/ubuntu/cephtest/apache/apache.client.0.conf' 2014-07-01 16:47:42,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 4 to mira0435601-218 2014-07-01 16:47:42,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 206 (ObjNum 1316 snap 0 seq_num 1316) dirty exists 2014-07-01 16:47:42,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: finishing write tid 1 to mira0435601-307 2014-07-01 16:47:42,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1989: done (15 left) 2014-07-01 16:47:42,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1991: done (14 left) 2014-07-01 16:47:42,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: done (13 left) 2014-07-01 16:47:42,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: done (12 left) 2014-07-01 16:47:42,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1996: done (11 left) 2014-07-01 16:47:42,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: done (10 left) 2014-07-01 16:47:42,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:1999: done (9 left) 2014-07-01 16:47:42,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: done (8 left) 2014-07-01 16:47:42,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: done (7 left) 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2004: done (6 left) 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2005: done (5 left) 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 450 current snap is 0 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: seq_num 1324 ranges {671744=536576,1208320=458752,1667072=421888,2088960=323584} 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: writing mira0435601-450 from 671744 to 1208320 tid 1 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: writing mira0435601-450 from 1208320 to 1667072 tid 2 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: writing mira0435601-450 from 1667072 to 2088960 tid 3 2014-07-01 16:47:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: writing mira0435601-450 from 2088960 to 2412544 tid 4 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2009: read oid 259 snap -1 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: finishing write tid 3 to mira0435601-307 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: finishing write tid 4 to mira0435601-307 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 208 (ObjNum 1321 snap 0 seq_num 1321) dirty exists 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 4 to mira0435601-487 2014-07-01 16:47:42,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 185 (ObjNum 1315 snap 0 seq_num 1315) dirty exists 2014-07-01 16:47:42,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: finishing write tid 1 to mira0435601-411 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 4 to mira0435601-308 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 249 (ObjNum 1314 snap 0 seq_num 1314) dirty exists 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: finishing write tid 2 to mira0435601-411 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: finishing write tid 3 to mira0435601-411 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: finishing write tid 5 to mira0435601-411 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: finishing write tid 6 to mira0435601-411 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 140 (ObjNum 1322 snap 0 seq_num 1322) dirty exists 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: finishing write tid 1 to mira0435601-327 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: finishing write tid 3 to mira0435601-327 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: finishing write tid 4 to mira0435601-327 2014-07-01 16:47:42,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 181 (ObjNum 1323 snap 0 seq_num 1323) dirty exists 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 1 to mira0435601-450 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 2 to mira0435601-450 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: done (6 left) 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: done (5 left) 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: done (4 left) 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: done (3 left) 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: done (2 left) 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2010: read oid 106 snap -1 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: seq_num 1325 ranges {5042176=253952} 2014-07-01 16:47:42,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: writing mira0435601-409 from 5042176 to 5296128 tid 1 2014-07-01 16:47:42,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 76 current snap is 0 2014-07-01 16:47:42,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: seq_num 1326 ranges {786432=729088,1515520=671744,2187264=770048,2957312=163840} 2014-07-01 16:47:42,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: writing mira0435601-76 from 786432 to 1515520 tid 1 2014-07-01 16:47:42,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: writing mira0435601-76 from 1515520 to 2187264 tid 2 2014-07-01 16:47:42,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: writing mira0435601-76 from 2187264 to 2957312 tid 3 2014-07-01 16:47:42,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: writing mira0435601-76 from 2957312 to 3121152 tid 4 2014-07-01 16:47:42,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: read oid 109 snap -1 2014-07-01 16:47:42,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 3 to mira0435601-450 2014-07-01 16:47:42,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 4 to mira0435601-450 2014-07-01 16:47:42,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 6 to mira0435601-450 2014-07-01 16:47:42,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: finishing write tid 7 to mira0435601-450 2014-07-01 16:47:42,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 450 v 214 (ObjNum 1324 snap 0 seq_num 1324) dirty exists 2014-07-01 16:47:42,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 1 to mira0435601-76 2014-07-01 16:47:42,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 2 to mira0435601-76 2014-07-01 16:47:42,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: done (5 left) 2014-07-01 16:47:42,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2014: read oid 394 snap -1 2014-07-01 16:47:42,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: read oid 269 snap -1 2014-07-01 16:47:42,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: read oid 165 snap -1 2014-07-01 16:47:42,730.730 INFO:teuthology.run_tasks:Running task swift... 2014-07-01 16:47:42,732.732 INFO:teuthology.task.swift:clients=['client.0'] 2014-07-01 16:47:42,732.732 INFO:teuthology.task.swift:Downloading swift... 2014-07-01 16:47:42,733.733 INFO:teuthology.orchestra.run.mira043:Running: 'git clone https://github.com/ceph/swift.git /home/ubuntu/cephtest/swift' 2014-07-01 16:47:42,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 428 current snap is 0 2014-07-01 16:47:42,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: seq_num 1327 ranges {397312=757760,1155072=401408} 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: writing mira0435601-428 from 397312 to 1155072 tid 1 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: writing mira0435601-428 from 1155072 to 1556480 tid 2 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: seq_num 1328 ranges {2420736=475136,2895872=487424,3383296=716800,4100096=401408} 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0435601-200 from 2420736 to 2895872 tid 1 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0435601-200 from 2895872 to 3383296 tid 2 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0435601-200 from 3383296 to 4100096 tid 3 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0435601-200 from 4100096 to 4501504 tid 4 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 119 current snap is 0 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: seq_num 1329 ranges {3604480=499712,4104192=401408} 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: writing mira0435601-119 from 3604480 to 4104192 tid 1 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: writing mira0435601-119 from 4104192 to 4505600 tid 2 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2020: delete oid 250 current snap is 0 2014-07-01 16:47:42,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 3 to mira0435601-76 2014-07-01 16:47:42,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 4 to mira0435601-76 2014-07-01 16:47:42,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 6 to mira0435601-76 2014-07-01 16:47:42,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: finishing write tid 7 to mira0435601-76 2014-07-01 16:47:42,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 248 (ObjNum 1326 snap 0 seq_num 1326) dirty exists 2014-07-01 16:47:42,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: expect (ObjNum 1148 snap 0 seq_num 1148) 2014-07-01 16:47:43,078.078 INFO:teuthology.orchestra.run.mira043.stdout:Cloning into '/home/ubuntu/cephtest/swift'... 2014-07-01 16:47:45,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: finishing write tid 1 to mira0435601-428 2014-07-01 16:47:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: finishing write tid 2 to mira0435601-428 2014-07-01 16:47:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: finishing write tid 4 to mira0435601-428 2014-07-01 16:47:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: finishing write tid 5 to mira0435601-428 2014-07-01 16:47:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 105 (ObjNum 1327 snap 0 seq_num 1327) dirty exists 2014-07-01 16:47:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 1 to mira0435601-200 2014-07-01 16:47:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 2 to mira0435601-200 2014-07-01 16:47:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 3 to mira0435601-200 2014-07-01 16:47:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 4 to mira0435601-200 2014-07-01 16:47:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 6 to mira0435601-200 2014-07-01 16:47:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 7 to mira0435601-200 2014-07-01 16:47:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 110 (ObjNum 1328 snap 0 seq_num 1328) dirty exists 2014-07-01 16:47:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2009: done (11 left) 2014-07-01 16:47:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: done (10 left) 2014-07-01 16:47:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: done (9 left) 2014-07-01 16:47:45,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: done (8 left) 2014-07-01 16:47:45,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: done (7 left) 2014-07-01 16:47:45,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2020: done (6 left) 2014-07-01 16:47:45,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: read oid 354 snap -1 2014-07-01 16:47:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: read oid 206 snap -1 2014-07-01 16:47:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 25 current snap is 0 2014-07-01 16:47:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: seq_num 1331 ranges {1208320=450560} 2014-07-01 16:47:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: writing mira0435601-25 from 1208320 to 1658880 tid 1 2014-07-01 16:47:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2024: read oid 416 snap -1 2014-07-01 16:47:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2025: read oid 173 snap -1 2014-07-01 16:47:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2026: delete oid 467 current snap is 0 2014-07-01 16:47:45,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: finishing write tid 1 to mira0435601-409 2014-07-01 16:47:45,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: finishing write tid 3 to mira0435601-409 2014-07-01 16:47:45,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2014: expect (ObjNum 821 snap 0 seq_num 821) 2014-07-01 16:47:45,942.942 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:46,535.535 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:48,826.826 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:48,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 1 to mira0435601-119 2014-07-01 16:47:48,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 2 to mira0435601-119 2014-07-01 16:47:48,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 4 to mira0435601-119 2014-07-01 16:47:48,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: expect (ObjNum 1123 snap 0 seq_num 1123) 2014-07-01 16:47:49,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: expect (ObjNum 942 snap 0 seq_num 942) 2014-07-01 16:47:49,506.506 INFO:teuthology.task.swift:Creating rgw users... 2014-07-01 16:47:49,507.507 INFO:teuthology.orchestra.run.mira043:Running: "adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --subuser foo.client.0:foo --display-name 'Mr. foo.client.0 foo' --secret Px0Ekt7AqeeiSv0oOe4YJft8KJDkjngu2dcsm2I8ZCEmvGaEIQmtKw== --email foo.client.0+test@test.test --key-type swift" 2014-07-01 16:47:49,565.565 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:46:54.537811', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886655244, u'health': u'HEALTH_OK', u'kb_used': 10125808, u'store_stats': {u'bytes_total': 12022729, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634105}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'3 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'8 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'42 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1768/13633 objects degraded (12.969%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 62027810, u'degraded_objects': 1768, u'num_pgs': 324, u'recovering_keys_per_sec': 0, u'data_bytes': 26902442605, u'degraded_total': 13633, u'bytes_used': 52291313664, u'recovering_objects_per_sec': 14, u'write_bytes_sec': 1009, u'op_per_sec': 38, u'version': 1505, u'pgs_by_state': [{u'count': 5, u'state_name': u'creating'}, {u'count': 3, u'state_name': u'peering'}, {u'count': 20, u'state_name': u'active+recovery_wait'}, {u'count': 282, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 11568630, u'degraded_ratio': u'12.969', u'bytes_avail': 4946291224576}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 609, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:47:49,566.566 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:47:50,360.360 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:47:50,361.361 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:51,178.178 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:51,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:2010: expect (ObjNum 903 snap 0 seq_num 903) 2014-07-01 16:47:51,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2024: expect (ObjNum 960 snap 0 seq_num 960) 2014-07-01 16:47:54,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: finishing write tid 1 to mira0435601-25 2014-07-01 16:47:54,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: expect (ObjNum 841 snap 0 seq_num 841) 2014-07-01 16:47:55,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: finishing write tid 3 to mira0435601-25 2014-07-01 16:47:55,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: expect (ObjNum 353 snap 0 seq_num 353) 2014-07-01 16:47:56,289.289 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:47:56,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2025: expect (ObjNum 1063 snap 0 seq_num 1063) 2014-07-01 16:47:57,038.038 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:47:57,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 5 to mira0435601-119 2014-07-01 16:47:57,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 228 (ObjNum 1329 snap 0 seq_num 1329) dirty exists 2014-07-01 16:47:57,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: finishing write tid 4 to mira0435601-409 2014-07-01 16:47:57,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 251 (ObjNum 1325 snap 0 seq_num 1325) dirty exists 2014-07-01 16:47:57,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: finishing write tid 4 to mira0435601-25 2014-07-01 16:47:57,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 252 (ObjNum 1331 snap 0 seq_num 1331) dirty exists 2014-07-01 16:47:57,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2010: done (11 left) 2014-07-01 16:47:57,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: done (10 left) 2014-07-01 16:47:57,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: done (9 left) 2014-07-01 16:47:57,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2014: done (8 left) 2014-07-01 16:47:57,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: done (7 left) 2014-07-01 16:47:57,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: done (6 left) 2014-07-01 16:47:57,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: done (5 left) 2014-07-01 16:47:57,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: done (4 left) 2014-07-01 16:47:57,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: done (3 left) 2014-07-01 16:47:57,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2024: done (2 left) 2014-07-01 16:47:57,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2025: done (1 left) 2014-07-01 16:47:57,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:2026: done (0 left) 2014-07-01 16:47:57,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: seq_num 1333 ranges {8327168=749568,9076736=802816} 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: writing mira0435601-439 from 8327168 to 9076736 tid 1 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: writing mira0435601-439 from 9076736 to 9879552 tid 2 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 251 current snap is 0 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: seq_num 1334 ranges {3702784=622592,4325376=634880,4960256=167936} 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: writing mira0435601-251 from 3702784 to 4325376 tid 1 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: writing mira0435601-251 from 4325376 to 4960256 tid 2 2014-07-01 16:47:57,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: writing mira0435601-251 from 4960256 to 5128192 tid 3 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: read oid 155 snap -1 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: read oid 326 snap -1 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: finishing write tid 1 to mira0435601-439 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: finishing write tid 2 to mira0435601-439 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: finishing write tid 4 to mira0435601-439 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: finishing write tid 5 to mira0435601-439 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 230 (ObjNum 1333 snap 0 seq_num 1333) dirty exists 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: done (3 left) 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2031: read oid 42 snap -1 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2032: read oid 117 snap -1 2014-07-01 16:47:57,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2033: delete oid 272 current snap is 0 2014-07-01 16:47:59,378.378 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:00,089.089 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'3 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'38 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'72 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1768/13619 objects degraded (12.982%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 1768, u'num_pgs': 356, u'data_bytes': 26873082477, u'degraded_total': 13619, u'bytes_used': 52075294720, u'op_per_sec': 1, u'write_bytes_sec': 0, u'version': 1512, u'pgs_by_state': [{u'count': 35, u'state_name': u'creating'}, {u'count': 3, u'state_name': u'peering'}, {u'count': 20, u'state_name': u'active+recovery_wait'}, {u'count': 284, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'degraded_ratio': u'12.982', u'bytes_avail': 4946507243520}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 615, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:00,089.089 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:00,895.895 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:48:00,896.896 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: finishing write tid 1 to mira0435601-251 2014-07-01 16:48:01,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: finishing write tid 2 to mira0435601-251 2014-07-01 16:48:01,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: expect (ObjNum 1176 snap 0 seq_num 1176) 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: finishing write tid 3 to mira0435601-251 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: finishing write tid 5 to mira0435601-251 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: done (5 left) 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: done (4 left) 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2032: done (3 left) 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2033: done (2 left) 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 6 current snap is 0 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: seq_num 1336 ranges {1073152=569344,1642496=401408} 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: writing mira0435601-6 from 1073152 to 1642496 tid 1 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: writing mira0435601-6 from 1642496 to 2043904 tid 2 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: read oid 201 snap -1 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: delete oid 372 current snap is 0 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 1 to mira0435601-6 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 2 to mira0435601-6 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 4 to mira0435601-6 2014-07-01 16:48:01,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: done (4 left) 2014-07-01 16:48:01,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 39 current snap is 0 2014-07-01 16:48:01,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: seq_num 1338 ranges {311296=712704} 2014-07-01 16:48:01,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: writing mira0435601-39 from 311296 to 1024000 tid 1 2014-07-01 16:48:01,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: read oid 64 snap -1 2014-07-01 16:48:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 5 to mira0435601-6 2014-07-01 16:48:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 104 (ObjNum 1336 snap 0 seq_num 1336) dirty exists 2014-07-01 16:48:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: done (5 left) 2014-07-01 16:48:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2039: read oid 260 snap -1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2040: read oid 281 snap -1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 10 current snap is 0 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: seq_num 1339 ranges {0=389120} 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: writing mira0435601-10 from 0 to 389120 tid 1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2042: read oid 446 snap -1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: read oid 40 snap -1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: read oid 149 snap -1 2014-07-01 16:48:01,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2045: delete oid 24 current snap is 0 2014-07-01 16:48:01,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: finishing write tid 6 to mira0435601-251 2014-07-01 16:48:01,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 182 (ObjNum 1334 snap 0 seq_num 1334) dirty exists 2014-07-01 16:48:01,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2031: expect (ObjNum 561 snap 0 seq_num 561) 2014-07-01 16:48:01,662.662 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:03,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2039: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 16:48:04,152.152 INFO:teuthology.orchestra.run.mira043.stdout:{ "user_id": "foo.client.0", 2014-07-01 16:48:04,152.152 INFO:teuthology.orchestra.run.mira043.stdout: "display_name": "Mr. foo.client.0 foo", 2014-07-01 16:48:04,153.153 INFO:teuthology.orchestra.run.mira043.stdout: "email": "foo.client.0+test@test.test", 2014-07-01 16:48:04,153.153 INFO:teuthology.orchestra.run.mira043.stdout: "suspended": 0, 2014-07-01 16:48:04,153.153 INFO:teuthology.orchestra.run.mira043.stdout: "max_buckets": 1000, 2014-07-01 16:48:04,153.153 INFO:teuthology.orchestra.run.mira043.stdout: "auid": 0, 2014-07-01 16:48:04,154.154 INFO:teuthology.orchestra.run.mira043.stdout: "subusers": [], 2014-07-01 16:48:04,154.154 INFO:teuthology.orchestra.run.mira043.stdout: "keys": [], 2014-07-01 16:48:04,154.154 INFO:teuthology.orchestra.run.mira043.stdout: "swift_keys": [ 2014-07-01 16:48:04,154.154 INFO:teuthology.orchestra.run.mira043.stdout: { "user": "foo.client.0:foo", 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "secret_key": "Px0Ekt7AqeeiSv0oOe4YJft8KJDkjngu2dcsm2I8ZCEmvGaEIQmtKw=="}], 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "caps": [], 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "op_mask": "read, write, delete", 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "default_placement": "", 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "placement_tags": [], 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "bucket_quota": { "enabled": false, 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "max_size_kb": -1, 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "max_objects": -1}, 2014-07-01 16:48:04,155.155 INFO:teuthology.orchestra.run.mira043.stdout: "user_quota": { "enabled": false, 2014-07-01 16:48:04,156.156 INFO:teuthology.orchestra.run.mira043.stdout: "max_size_kb": -1, 2014-07-01 16:48:04,156.156 INFO:teuthology.orchestra.run.mira043.stdout: "max_objects": -1}, 2014-07-01 16:48:04,156.156 INFO:teuthology.orchestra.run.mira043.stdout: "temp_url_keys": []} 2014-07-01 16:48:05,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2042: expect (ObjNum 1287 snap 0 seq_num 1287) 2014-07-01 16:48:05,937.937 INFO:teuthology.orchestra.run.mira043:Running: "adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --subuser bar.client.0:bar --display-name 'Mr. bar.client.0 bar' --secret i6t5X2wDvkfWtYKdgOvLfGeUYE/JRThW077lcUSOrnMoCMzi6ZfDWg== --email bar.client.0+test@test.test --key-type swift" 2014-07-01 16:48:06,876.876 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:07,646.646 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:08,762.762 INFO:teuthology.orchestra.run.mira043.stdout:{ "user_id": "bar.client.0", 2014-07-01 16:48:08,763.763 INFO:teuthology.orchestra.run.mira043.stdout: "display_name": "Mr. bar.client.0 bar", 2014-07-01 16:48:08,763.763 INFO:teuthology.orchestra.run.mira043.stdout: "email": "bar.client.0+test@test.test", 2014-07-01 16:48:08,764.764 INFO:teuthology.orchestra.run.mira043.stdout: "suspended": 0, 2014-07-01 16:48:08,764.764 INFO:teuthology.orchestra.run.mira043.stdout: "max_buckets": 1000, 2014-07-01 16:48:08,764.764 INFO:teuthology.orchestra.run.mira043.stdout: "auid": 0, 2014-07-01 16:48:08,765.765 INFO:teuthology.orchestra.run.mira043.stdout: "subusers": [], 2014-07-01 16:48:08,765.765 INFO:teuthology.orchestra.run.mira043.stdout: "keys": [], 2014-07-01 16:48:08,765.765 INFO:teuthology.orchestra.run.mira043.stdout: "swift_keys": [ 2014-07-01 16:48:08,766.766 INFO:teuthology.orchestra.run.mira043.stdout: { "user": "bar.client.0:bar", 2014-07-01 16:48:08,766.766 INFO:teuthology.orchestra.run.mira043.stdout: "secret_key": "i6t5X2wDvkfWtYKdgOvLfGeUYE\/JRThW077lcUSOrnMoCMzi6ZfDWg=="}], 2014-07-01 16:48:08,766.766 INFO:teuthology.orchestra.run.mira043.stdout: "caps": [], 2014-07-01 16:48:08,766.766 INFO:teuthology.orchestra.run.mira043.stdout: "op_mask": "read, write, delete", 2014-07-01 16:48:08,767.767 INFO:teuthology.orchestra.run.mira043.stdout: "default_placement": "", 2014-07-01 16:48:08,767.767 INFO:teuthology.orchestra.run.mira043.stdout: "placement_tags": [], 2014-07-01 16:48:08,767.767 INFO:teuthology.orchestra.run.mira043.stdout: "bucket_quota": { "enabled": false, 2014-07-01 16:48:08,768.768 INFO:teuthology.orchestra.run.mira043.stdout: "max_size_kb": -1, 2014-07-01 16:48:08,768.768 INFO:teuthology.orchestra.run.mira043.stdout: "max_objects": -1}, 2014-07-01 16:48:08,768.768 INFO:teuthology.orchestra.run.mira043.stdout: "user_quota": { "enabled": false, 2014-07-01 16:48:08,769.769 INFO:teuthology.orchestra.run.mira043.stdout: "max_size_kb": -1, 2014-07-01 16:48:08,769.769 INFO:teuthology.orchestra.run.mira043.stdout: "max_objects": -1}, 2014-07-01 16:48:08,769.769 INFO:teuthology.orchestra.run.mira043.stdout: "temp_url_keys": []} 2014-07-01 16:48:09,165.165 INFO:teuthology.task.swift:Configuring testswift... 2014-07-01 16:48:09,165.165 INFO:teuthology.task.swift:client=client.0 2014-07-01 16:48:09,166.166 INFO:teuthology.task.swift:config={'clients': {'client.0': {'rgw_server': 'client.0'}}, 'testswift_conf': {'client.0': ConfigObj({'func_test': {'auth_port': 7280, 'auth_prefix': '/auth/', 'auth_ssl': 'no', 'account': 'foo.client.0', 'username': 'foo', 'email': 'foo.client.0+test@test.test', 'display_name': 'Mr. foo.client.0 foo', 'password': 'Px0Ekt7AqeeiSv0oOe4YJft8KJDkjngu2dcsm2I8ZCEmvGaEIQmtKw==', 'account2': 'bar.client.0', 'username2': 'bar', 'email2': 'bar.client.0+test@test.test', 'display_name2': 'Mr. bar.client.0 bar', 'password2': 'i6t5X2wDvkfWtYKdgOvLfGeUYE/JRThW077lcUSOrnMoCMzi6ZfDWg=='}})}} 2014-07-01 16:48:09,166.166 INFO:teuthology.task.swift:roles: ['mon.a', 'mon.b', 'mds.a', 'osd.0', 'osd.1', 'osd.2'] 2014-07-01 16:48:09,166.166 INFO:teuthology.task.swift:target: ubuntu@mira079.front.sepia.ceph.com 2014-07-01 16:48:09,166.166 INFO:teuthology.task.swift:roles: ['osd.3', 'osd.4', 'osd.5', 'mon.c'] 2014-07-01 16:48:09,166.166 INFO:teuthology.task.swift:target: ubuntu@mira028.front.sepia.ceph.com 2014-07-01 16:48:09,167.167 INFO:teuthology.task.swift:roles: ['client.0'] 2014-07-01 16:48:09,167.167 INFO:teuthology.task.swift:target: ubuntu@mira043.front.sepia.ceph.com 2014-07-01 16:48:09,167.167 INFO:teuthology.task.swift:client.0 2014-07-01 16:48:09,167.167 INFO:teuthology.orchestra.run.mira043:Running: 'cd /home/ubuntu/cephtest/swift && ./bootstrap' 2014-07-01 16:48:09,954.954 INFO:teuthology.orchestra.run.mira043.stdout:The --no-site-packages flag is deprecated; it is now the default behavior. 2014-07-01 16:48:09,954.954 INFO:teuthology.orchestra.run.mira043.stdout:New python executable in virtualenv/bin/python 2014-07-01 16:48:10,033.033 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: finishing write tid 1 to mira0435601-39 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: finishing write tid 3 to mira0435601-39 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: finishing write tid 4 to mira0435601-39 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 39 v 217 (ObjNum 1338 snap 0 seq_num 1338) dirty exists 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: done (11 left) 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2031: done (10 left) 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: done (9 left) 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2039: done (8 left) 2014-07-01 16:48:10,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2042: done (7 left) 2014-07-01 16:48:10,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2045: done (6 left) 2014-07-01 16:48:10,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2046: read oid 312 snap -1 2014-07-01 16:48:10,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: expect (ObjNum 200 snap 0 seq_num 200) 2014-07-01 16:48:10,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: expect (ObjNum 1152 snap 0 seq_num 1152) 2014-07-01 16:48:10,756.756 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:47:11.320775', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886647896, u'health': u'HEALTH_OK', u'kb_used': 10133156, u'store_stats': {u'bytes_total': 13071347, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3634147}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'14 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'33 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1502/13155 objects degraded (11.418%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 41726355, u'degraded_objects': 1502, u'num_pgs': 364, u'recovering_keys_per_sec': 0, u'data_bytes': 25736471981, u'degraded_total': 13155, u'bytes_used': 50158510080, u'recovering_objects_per_sec': 9, u'write_bytes_sec': 130394, u'op_per_sec': 47, u'version': 1521, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 331, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+recovering'}], u'read_bytes_sec': 8846476, u'degraded_ratio': u'11.418', u'bytes_avail': 4948424028160}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 619, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:10,757.757 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:10,887.887 INFO:teuthology.orchestra.run.mira043.stdout:Installing distribute.............................................................................................................................................................................................done. 2014-07-01 16:48:11,272.272 INFO:teuthology.orchestra.run.mira043.stdout:Installing pip...............done. 2014-07-01 16:48:11,539.539 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:48:11,539.539 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:11,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2040: expect (ObjNum 1107 snap 0 seq_num 1107) 2014-07-01 16:48:12,351.351 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:12,484.484 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking pip from https://pypi.python.org/packages/source/p/pip/pip-1.5.6.tar.gz#md5=01026f87978932060cc86c1dc527903e 2014-07-01 16:48:12,485.485 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py egg_info for package pip 2014-07-01 16:48:12,485.485 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:12,485.485 INFO:teuthology.orchestra.run.mira043.stdout: warning: no files found matching 'pip/cacert.pem' 2014-07-01 16:48:12,486.486 INFO:teuthology.orchestra.run.mira043.stdout: warning: no files found matching '*.html' under directory 'docs' 2014-07-01 16:48:12,486.486 INFO:teuthology.orchestra.run.mira043.stdout: warning: no previously-included files matching '*.rst' found under directory 'docs/_build' 2014-07-01 16:48:12,486.486 INFO:teuthology.orchestra.run.mira043.stdout: no previously-included directories found matching 'docs/_build/_sources' 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout:Installing collected packages: pip 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: Found existing installation: pip 1.1 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: Uninstalling pip: 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: Successfully uninstalled pip 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for pip 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: warning: no files found matching 'pip/cacert.pem' 2014-07-01 16:48:12,487.487 INFO:teuthology.orchestra.run.mira043.stdout: warning: no files found matching '*.html' under directory 'docs' 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout: warning: no previously-included files matching '*.rst' found under directory 'docs/_build' 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout: no previously-included directories found matching 'docs/_build/_sources' 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout: Installing pip script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout: Installing pip2.7 script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout: Installing pip2 script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout:Successfully installed pip 2014-07-01 16:48:12,488.488 INFO:teuthology.orchestra.run.mira043.stdout:Cleaning up... 2014-07-01 16:48:12,774.774 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking setuptools 2014-07-01 16:48:14,836.836 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/setuptools/setup.py) egg_info for package setuptools 2014-07-01 16:48:15,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: finishing write tid 1 to mira0435601-10 2014-07-01 16:48:15,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: finishing write tid 3 to mira0435601-10 2014-07-01 16:48:15,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 16:48:15,037.037 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:15,087.087 INFO:teuthology.orchestra.run.mira043.stdout:Installing collected packages: setuptools 2014-07-01 16:48:15,088.088 INFO:teuthology.orchestra.run.mira043.stdout: Found existing installation: distribute 0.6.24 2014-07-01 16:48:15,090.090 INFO:teuthology.orchestra.run.mira043.stdout: Uninstalling distribute: 2014-07-01 16:48:15,183.183 INFO:teuthology.orchestra.run.mira043.stdout: Successfully uninstalled distribute 2014-07-01 16:48:15,184.184 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for setuptools 2014-07-01 16:48:15,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: finishing write tid 4 to mira0435601-10 2014-07-01 16:48:15,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 10 v 95 (ObjNum 1339 snap 0 seq_num 1339) dirty exists 2014-07-01 16:48:15,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: expect (ObjNum 1239 snap 0 seq_num 1239) 2014-07-01 16:48:15,383.383 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:15,447.447 INFO:teuthology.orchestra.run.mira043.stdout: Installing easy_install script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:15,448.448 INFO:teuthology.orchestra.run.mira043.stdout: Installing easy_install-2.7 script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:15,474.474 INFO:teuthology.orchestra.run.mira043.stdout:Successfully installed setuptools 2014-07-01 16:48:15,474.474 INFO:teuthology.orchestra.run.mira043.stdout:Cleaning up... 2014-07-01 16:48:15,781.781 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking PyYAML (from -r requirements.txt (line 1)) 2014-07-01 16:48:16,046.046 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/PyYAML/setup.py) egg_info for package PyYAML 2014-07-01 16:48:16,154.154 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:16,185.185 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking nose>=1.0.0 (from -r requirements.txt (line 2)) 2014-07-01 16:48:16,350.350 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/nose/setup.py) egg_info for package nose 2014-07-01 16:48:16,472.472 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:16,491.491 INFO:teuthology.orchestra.run.mira043.stdout: no previously-included directories found matching 'doc/.build' 2014-07-01 16:48:16,514.514 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking boto>=2.0b4 (from -r requirements.txt (line 3)) 2014-07-01 16:48:16,717.717 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking bunch>=1.0.0 (from -r requirements.txt (line 4)) 2014-07-01 16:48:16,732.732 INFO:teuthology.orchestra.run.mira043.stdout: Downloading bunch-1.0.1.tar.gz 2014-07-01 16:48:16,736.736 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/bunch/setup.py) egg_info for package bunch 2014-07-01 16:48:16,831.831 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:16,854.854 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking gevent==0.13.6 (from -r requirements.txt (line 6)) 2014-07-01 16:48:17,081.081 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/gevent/setup.py) egg_info for package gevent 2014-07-01 16:48:17,181.181 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:17,220.220 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking isodate>=0.4.4 (from -r requirements.txt (line 7)) 2014-07-01 16:48:17,240.240 INFO:teuthology.orchestra.run.mira043.stdout: Downloading isodate-0.5.0.tar.gz 2014-07-01 16:48:17,251.251 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/isodate/setup.py) egg_info for package isodate 2014-07-01 16:48:17,346.346 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:17,372.372 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking eventlet>=0.9.14 (from -r requirements.txt (line 8)) 2014-07-01 16:48:17,503.503 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/eventlet/setup.py) egg_info for package eventlet 2014-07-01 16:48:17,609.609 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:17,646.646 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking netifaces>=0.5 (from -r requirements.txt (line 9)) 2014-07-01 16:48:17,675.675 INFO:teuthology.orchestra.run.mira043.stdout: Downloading netifaces-0.10.4.tar.gz 2014-07-01 16:48:17,681.681 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/netifaces/setup.py) egg_info for package netifaces 2014-07-01 16:48:17,711.711 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:17,788.788 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:17,819.819 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking xattr>=0.6,<0.7.0 (from -r requirements.txt (line 10)) 2014-07-01 16:48:17,839.839 INFO:teuthology.orchestra.run.mira043.stdout: Downloading xattr-0.6.4.tar.gz 2014-07-01 16:48:17,847.847 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/xattr/setup.py) egg_info for package xattr 2014-07-01 16:48:17,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2046: expect (ObjNum 863 snap 0 seq_num 863) 2014-07-01 16:48:17,955.955 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:17,985.985 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking simplejson>=2.1.3 (from -r requirements.txt (line 11)) 2014-07-01 16:48:18,046.046 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/simplejson/setup.py) egg_info for package simplejson 2014-07-01 16:48:18,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: done (6 left) 2014-07-01 16:48:18,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: done (5 left) 2014-07-01 16:48:18,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2040: done (4 left) 2014-07-01 16:48:18,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: done (3 left) 2014-07-01 16:48:18,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: done (2 left) 2014-07-01 16:48:18,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: done (1 left) 2014-07-01 16:48:18,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2046: done (0 left) 2014-07-01 16:48:18,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 16:48:18,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: seq_num 1341 ranges {2551808=131072} 2014-07-01 16:48:18,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: writing mira0435601-296 from 2551808 to 2682880 tid 1 2014-07-01 16:48:18,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 82 current snap is 0 2014-07-01 16:48:18,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: seq_num 1342 ranges {3137536=561152,3698688=438272,4136960=364544} 2014-07-01 16:48:18,151.151 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:18,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: writing mira0435601-82 from 3137536 to 3698688 tid 1 2014-07-01 16:48:18,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: writing mira0435601-82 from 3698688 to 4136960 tid 2 2014-07-01 16:48:18,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: writing mira0435601-82 from 4136960 to 4501504 tid 3 2014-07-01 16:48:18,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 453 current snap is 0 2014-07-01 16:48:18,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: seq_num 1343 ranges {1953792=434176,2387968=425984,2813952=376832} 2014-07-01 16:48:18,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0435601-453 from 1953792 to 2387968 tid 1 2014-07-01 16:48:18,214.214 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking WebOb==1.0 (from -r requirements.txt (line 12)) 2014-07-01 16:48:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0435601-453 from 2387968 to 2813952 tid 2 2014-07-01 16:48:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0435601-453 from 2813952 to 3190784 tid 3 2014-07-01 16:48:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 192 current snap is 0 2014-07-01 16:48:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: seq_num 1344 ranges {1093632=507904,1601536=401408} 2014-07-01 16:48:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: writing mira0435601-192 from 1093632 to 1601536 tid 1 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: writing mira0435601-192 from 1601536 to 2002944 tid 2 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2051: read oid 89 snap -1 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: finishing write tid 1 to mira0435601-82 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: finishing write tid 2 to mira0435601-82 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: finishing write tid 3 to mira0435601-82 2014-07-01 16:48:18,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: finishing write tid 5 to mira0435601-82 2014-07-01 16:48:18,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: finishing write tid 6 to mira0435601-82 2014-07-01 16:48:18,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 317 (ObjNum 1342 snap 0 seq_num 1342) dirty exists 2014-07-01 16:48:18,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: done (4 left) 2014-07-01 16:48:18,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 316 current snap is 0 2014-07-01 16:48:18,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: seq_num 1345 ranges {1949696=782336} 2014-07-01 16:48:18,277.277 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:18,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: writing mira0435601-316 from 1949696 to 2732032 tid 1 2014-07-01 16:48:18,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: read oid 180 snap -1 2014-07-01 16:48:18,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 401 current snap is 0 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: seq_num 1346 ranges {0=651264} 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: writing mira0435601-401 from 0 to 651264 tid 1 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2055: read oid 207 snap -1 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2056: read oid 131 snap -1 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 193 current snap is 0 2014-07-01 16:48:18,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: seq_num 1347 ranges {3235840=679936,3915776=651264,4567040=552960} 2014-07-01 16:48:18,380.380 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/WebOb/setup.py) egg_info for package WebOb 2014-07-01 16:48:18,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: writing mira0435601-193 from 3235840 to 3915776 tid 1 2014-07-01 16:48:18,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: writing mira0435601-193 from 3915776 to 4567040 tid 2 2014-07-01 16:48:18,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: writing mira0435601-193 from 4567040 to 5120000 tid 3 2014-07-01 16:48:18,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 340 current snap is 0 2014-07-01 16:48:18,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: seq_num 1348 ranges {1458176=200704} 2014-07-01 16:48:18,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: writing mira0435601-340 from 1458176 to 1658880 tid 1 2014-07-01 16:48:18,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2059: read oid 169 snap -1 2014-07-01 16:48:18,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2056: expect (ObjNum 815 snap 0 seq_num 815) 2014-07-01 16:48:18,477.477 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:18,477.477 INFO:teuthology.orchestra.run.mira043.stdout: warning: no previously-included files matching '*' found under directory 'docs/_build/_sources' 2014-07-01 16:48:18,477.477 INFO:teuthology.orchestra.run.mira043.stdout:Downloading/unpacking greenlet (from gevent==0.13.6->-r requirements.txt (line 6)) 2014-07-01 16:48:18,549.549 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py (path:/home/ubuntu/cephtest/swift/virtualenv/build/greenlet/setup.py) egg_info for package greenlet 2014-07-01 16:48:18,668.668 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:18,693.693 INFO:teuthology.orchestra.run.mira043.stdout:Installing collected packages: PyYAML, nose, boto, bunch, gevent, isodate, eventlet, netifaces, xattr, simplejson, WebOb, greenlet 2014-07-01 16:48:18,694.694 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for PyYAML 2014-07-01 16:48:18,794.794 INFO:teuthology.orchestra.run.mira043.stdout: checking if libyaml is compilable 2014-07-01 16:48:18,795.795 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.7 -c build/temp.linux-x86_64-2.7/check_libyaml.c -o build/temp.linux-x86_64-2.7/check_libyaml.o 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: build/temp.linux-x86_64-2.7/check_libyaml.c:2:18: fatal error: yaml.h: No such file or directory 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: compilation terminated. 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: libyaml is not found or a compiler error: forcing --without-libyaml 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: (if libyaml is installed correctly, you may need to 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: specify the option --include-dirs or uncomment and 2014-07-01 16:48:18,833.833 INFO:teuthology.orchestra.run.mira043.stdout: modify the parameter include_dirs in setup.cfg) 2014-07-01 16:48:18,892.892 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:18,925.925 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for nose 2014-07-01 16:48:19,166.166 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:19,191.191 INFO:teuthology.orchestra.run.mira043.stdout: no previously-included directories found matching 'doc/.build' 2014-07-01 16:48:19,213.213 INFO:teuthology.orchestra.run.mira043.stdout: Installing nosetests script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:19,214.214 INFO:teuthology.orchestra.run.mira043.stdout: Installing nosetests-2.7 script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:19,943.943 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for bunch 2014-07-01 16:48:19,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2059: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 16:48:20,054.054 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:20,083.083 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for gevent 2014-07-01 16:48:20,195.195 INFO:teuthology.orchestra.run.mira043.stdout: building 'gevent.core' extension 2014-07-01 16:48:20,195.195 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.7 -c gevent/core.c -o build/temp.linux-x86_64-2.7/gevent/core.o 2014-07-01 16:48:20,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: finishing write tid 1 to mira0435601-296 2014-07-01 16:48:20,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: finishing write tid 3 to mira0435601-296 2014-07-01 16:48:20,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: finishing write tid 1 to mira0435601-401 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: finishing write tid 3 to mira0435601-401 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 1 to mira0435601-453 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: finishing write tid 4 to mira0435601-401 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 103 (ObjNum 1346 snap 0 seq_num 1346) dirty exists 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 2 to mira0435601-453 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 3 to mira0435601-453 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 5 to mira0435601-453 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: finishing write tid 1 to mira0435601-192 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: finishing write tid 2 to mira0435601-192 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: finishing write tid 4 to mira0435601-192 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: finishing write tid 1 to mira0435601-316 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: finishing write tid 3 to mira0435601-316 2014-07-01 16:48:20,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: finishing write tid 4 to mira0435601-296 2014-07-01 16:48:20,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 184 (ObjNum 1341 snap 0 seq_num 1341) dirty exists 2014-07-01 16:48:20,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 6 to mira0435601-453 2014-07-01 16:48:20,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 231 (ObjNum 1343 snap 0 seq_num 1343) dirty exists 2014-07-01 16:48:20,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2051: expect (ObjNum 600 snap 0 seq_num 600) 2014-07-01 16:48:20,605.605 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c: In function ‘__pyx_pf_6gevent_4core_12active_event_1add’: 2014-07-01 16:48:20,606.606 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c:5220:13: warning: variable ‘__pyx_v_timeout’ set but not used [-Wunused-but-set-variable] 2014-07-01 16:48:20,623.623 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c: In function ‘__pyx_pf_6gevent_4core_6buffer_7readline’: 2014-07-01 16:48:20,623.623 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c:7765:13: warning: variable ‘__pyx_v_size’ set but not used [-Wunused-but-set-variable] 2014-07-01 16:48:20,624.624 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c: In function ‘__pyx_pf_6gevent_4core_6buffer_8readlines’: 2014-07-01 16:48:20,771.771 INFO:teuthology.orchestra.run.mira043.stdout: gevent/core.c:8109:13: warning: variable ‘__pyx_v_hint’ set but not used [-Wunused-but-set-variable] 2014-07-01 16:48:20,882.882 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:21,596.596 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:48:11.321076', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886628372, u'health': u'HEALTH_OK', u'kb_used': 10152680, u'store_stats': {u'bytes_total': 6336854, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'32 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1370/12684 objects degraded (10.801%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 16353778, u'degraded_objects': 1370, u'num_pgs': 364, u'recovering_keys_per_sec': 0, u'data_bytes': 24729355709, u'degraded_total': 12684, u'bytes_used': 48368812032, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 1205771, u'op_per_sec': 19, u'version': 1530, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 332, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1330, u'degraded_ratio': u'10.801', u'bytes_avail': 4950213726208}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 619, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:21,597.597 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:22,374.374 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:48:22,374.374 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:23,207.207 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:23,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: finishing write tid 1 to mira0435601-193 2014-07-01 16:48:23,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2055: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:48:24,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: expect (ObjNum 179 snap 0 seq_num 179) 2014-07-01 16:48:24,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: finishing write tid 2 to mira0435601-193 2014-07-01 16:48:24,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: finishing write tid 3 to mira0435601-193 2014-07-01 16:48:24,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: finishing write tid 5 to mira0435601-193 2014-07-01 16:48:24,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: finishing write tid 5 to mira0435601-192 2014-07-01 16:48:24,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 234 (ObjNum 1344 snap 0 seq_num 1344) dirty exists 2014-07-01 16:48:24,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: finishing write tid 6 to mira0435601-193 2014-07-01 16:48:24,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 255 (ObjNum 1347 snap 0 seq_num 1347) dirty exists 2014-07-01 16:48:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: finishing write tid 4 to mira0435601-316 2014-07-01 16:48:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 181 (ObjNum 1345 snap 0 seq_num 1345) dirty exists 2014-07-01 16:48:24,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: finishing write tid 1 to mira0435601-340 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: finishing write tid 3 to mira0435601-340 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: finishing write tid 4 to mira0435601-340 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 187 (ObjNum 1348 snap 0 seq_num 1348) dirty exists 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: done (11 left) 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: done (10 left) 2014-07-01 16:48:24,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: done (9 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2051: done (8 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: done (7 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: done (6 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: done (5 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2055: done (4 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2056: done (3 left) 2014-07-01 16:48:24,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: done (2 left) 2014-07-01 16:48:24,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: done (1 left) 2014-07-01 16:48:24,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2059: done (0 left) 2014-07-01 16:48:24,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 398 current snap is 0 2014-07-01 16:48:24,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: seq_num 1349 ranges {2400256=69632} 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: writing mira0435601-398 from 2400256 to 2469888 tid 1 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 317 current snap is 0 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: seq_num 1350 ranges {0=499712,499712=647168,1146880=155648} 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: writing mira0435601-317 from 0 to 499712 tid 1 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: writing mira0435601-317 from 499712 to 1146880 tid 2 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: writing mira0435601-317 from 1146880 to 1302528 tid 3 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: read oid 432 snap -1 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 1 to mira0435601-398 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 3 to mira0435601-398 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 4 to mira0435601-398 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 106 (ObjNum 1349 snap 0 seq_num 1349) dirty exists 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: done (2 left) 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 153 current snap is 0 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: seq_num 1351 ranges {647168=618496,1265664=630784,1896448=724992,2621440=249856} 2014-07-01 16:48:24,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: writing mira0435601-153 from 647168 to 1265664 tid 1 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: writing mira0435601-153 from 1265664 to 1896448 tid 2 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: writing mira0435601-153 from 1896448 to 2621440 tid 3 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: writing mira0435601-153 from 2621440 to 2871296 tid 4 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 362 current snap is 0 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: seq_num 1352 ranges {3895296=643072,4538368=401408} 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: writing mira0435601-362 from 3895296 to 4538368 tid 1 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: writing mira0435601-362 from 4538368 to 4939776 tid 2 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 16:48:24,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: seq_num 1353 ranges {5926912=720896} 2014-07-01 16:48:24,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: writing mira0435601-238 from 5926912 to 6647808 tid 1 2014-07-01 16:48:24,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: read oid 231 snap -1 2014-07-01 16:48:24,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: expect (ObjNum 1269 snap 0 seq_num 1269) 2014-07-01 16:48:24,802.802 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro build/temp.linux-x86_64-2.7/gevent/core.o -levent -o build/lib.linux-x86_64-2.7/gevent/core.so 2014-07-01 16:48:24,845.845 INFO:teuthology.orchestra.run.mira043.stdout: Linking /home/ubuntu/cephtest/swift/virtualenv/build/gevent/build/lib.linux-x86_64-2.7/gevent/core.so to /home/ubuntu/cephtest/swift/virtualenv/build/gevent/gevent/core.so 2014-07-01 16:48:24,932.932 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:24,961.961 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for isodate 2014-07-01 16:48:25,098.098 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:25,130.130 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for eventlet 2014-07-01 16:48:25,373.373 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:25,429.429 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for netifaces 2014-07-01 16:48:25,576.576 INFO:teuthology.orchestra.run.mira043.stdout: checking for getifaddrs...found. 2014-07-01 16:48:25,609.609 INFO:teuthology.orchestra.run.mira043.stdout: checking for getnameinfo...found. 2014-07-01 16:48:25,996.996 INFO:teuthology.orchestra.run.mira043.stdout: checking for optional header files...netash/ash.h netatalk/at.h netax25/ax25.h neteconet/ec.h netipx/ipx.h netpacket/packet.h linux/irda.h linux/atm.h linux/llc.h linux/tipc.h linux/dn.h. 2014-07-01 16:48:26,012.012 INFO:teuthology.orchestra.run.mira043.stdout: checking whether struct sockaddr has a length field...no. 2014-07-01 16:48:26,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: finishing write tid 1 to mira0435601-362 2014-07-01 16:48:26,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: finishing write tid 2 to mira0435601-362 2014-07-01 16:48:26,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: finishing write tid 4 to mira0435601-362 2014-07-01 16:48:26,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: finishing write tid 5 to mira0435601-362 2014-07-01 16:48:26,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 248 (ObjNum 1352 snap 0 seq_num 1352) dirty exists 2014-07-01 16:48:26,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: expect (ObjNum 1091 snap 0 seq_num 1091) 2014-07-01 16:48:26,723.723 INFO:teuthology.orchestra.run.mira043.stdout: checking which sockaddr_xxx structs are defined...at ax25 in in6 ipx un ash ec ll atmpvc atmsvc dn irda llc. 2014-07-01 16:48:26,741.741 INFO:teuthology.orchestra.run.mira043.stdout: checking for routing socket support...no. 2014-07-01 16:48:26,761.761 INFO:teuthology.orchestra.run.mira043.stdout: checking for sysctl(CTL_NET...) support...no. 2014-07-01 16:48:26,791.791 INFO:teuthology.orchestra.run.mira043.stdout: checking for netlink support...yes. 2014-07-01 16:48:26,821.821 INFO:teuthology.orchestra.run.mira043.stdout: will use netlink to read routing table 2014-07-01 16:48:26,821.821 INFO:teuthology.orchestra.run.mira043.stdout: building 'netifaces' extension 2014-07-01 16:48:26,821.821 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -DNETIFACES_VERSION=0.10.4 -DHAVE_GETIFADDRS=1 -DHAVE_GETNAMEINFO=1 -DHAVE_NETASH_ASH_H=1 -DHAVE_NETATALK_AT_H=1 -DHAVE_NETAX25_AX25_H=1 -DHAVE_NETECONET_EC_H=1 -DHAVE_NETIPX_IPX_H=1 -DHAVE_NETPACKET_PACKET_H=1 -DHAVE_LINUX_IRDA_H=1 -DHAVE_LINUX_ATM_H=1 -DHAVE_LINUX_LLC_H=1 -DHAVE_LINUX_TIPC_H=1 -DHAVE_LINUX_DN_H=1 -DHAVE_SOCKADDR_AT=1 -DHAVE_SOCKADDR_AX25=1 -DHAVE_SOCKADDR_IN=1 -DHAVE_SOCKADDR_IN6=1 -DHAVE_SOCKADDR_IPX=1 -DHAVE_SOCKADDR_UN=1 -DHAVE_SOCKADDR_ASH=1 -DHAVE_SOCKADDR_EC=1 -DHAVE_SOCKADDR_LL=1 -DHAVE_SOCKADDR_ATMPVC=1 -DHAVE_SOCKADDR_ATMSVC=1 -DHAVE_SOCKADDR_DN=1 -DHAVE_SOCKADDR_IRDA=1 -DHAVE_SOCKADDR_LLC=1 -DHAVE_PF_NETLINK=1 -I/usr/include/python2.7 -c netifaces.c -o build/temp.linux-x86_64-2.7/netifaces.o 2014-07-01 16:48:27,048.048 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro build/temp.linux-x86_64-2.7/netifaces.o -o build/lib.linux-x86_64-2.7/netifaces.so 2014-07-01 16:48:27,087.087 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:27,117.117 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for xattr 2014-07-01 16:48:27,219.219 INFO:teuthology.orchestra.run.mira043.stdout: building 'xattr._xattr' extension 2014-07-01 16:48:27,219.219 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.7 -c xattr/_xattr.c -o build/temp.linux-x86_64-2.7/xattr/_xattr.o 2014-07-01 16:48:27,283.283 INFO:teuthology.orchestra.run.mira043.stdout: xattr/_xattr.c: In function ‘xattr_fsetxattr’: 2014-07-01 16:48:27,284.284 INFO:teuthology.orchestra.run.mira043.stdout: xattr/_xattr.c:497:9: warning: variable ‘nofollow’ set but not used [-Wunused-but-set-variable] 2014-07-01 16:48:27,285.285 INFO:teuthology.orchestra.run.mira043.stdout: xattr/_xattr.c: In function ‘init_xattr’: 2014-07-01 16:48:27,384.384 INFO:teuthology.orchestra.run.mira043.stdout: xattr/_xattr.c:944:15: warning: variable ‘m’ set but not used [-Wunused-but-set-variable] 2014-07-01 16:48:27,436.436 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro build/temp.linux-x86_64-2.7/xattr/_xattr.o -o build/lib.linux-x86_64-2.7/xattr/_xattr.so 2014-07-01 16:48:27,436.436 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:27,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 1 to mira0435601-238 2014-07-01 16:48:27,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 1 to mira0435601-153 2014-07-01 16:48:27,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 3 to mira0435601-238 2014-07-01 16:48:27,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 2 to mira0435601-153 2014-07-01 16:48:27,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 4 to mira0435601-238 2014-07-01 16:48:27,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 183 (ObjNum 1353 snap 0 seq_num 1353) dirty exists 2014-07-01 16:48:27,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 1 to mira0435601-317 2014-07-01 16:48:27,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 3 to mira0435601-153 2014-07-01 16:48:27,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 2 to mira0435601-317 2014-07-01 16:48:27,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 3 to mira0435601-317 2014-07-01 16:48:27,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 4 to mira0435601-153 2014-07-01 16:48:27,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: done (5 left) 2014-07-01 16:48:27,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: done (4 left) 2014-07-01 16:48:27,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: done (3 left) 2014-07-01 16:48:27,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: done (2 left) 2014-07-01 16:48:27,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 16:48:27,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: seq_num 1354 ranges {10231808=593920} 2014-07-01 16:48:27,456.456 INFO:teuthology.orchestra.run.mira043.stdout: Installing xattr script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:27,466.466 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for simplejson 2014-07-01 16:48:27,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: writing mira0435601-128 from 10231808 to 10825728 tid 1 2014-07-01 16:48:27,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 469 current snap is 0 2014-07-01 16:48:27,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: seq_num 1355 ranges {3203072=462848} 2014-07-01 16:48:27,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: writing mira0435601-469 from 3203072 to 3665920 tid 1 2014-07-01 16:48:27,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 115 current snap is 0 2014-07-01 16:48:27,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: seq_num 1356 ranges {0=491520,491520=688128,1179648=442368,1622016=73728} 2014-07-01 16:48:27,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: writing mira0435601-115 from 0 to 491520 tid 1 2014-07-01 16:48:27,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: writing mira0435601-115 from 491520 to 1179648 tid 2 2014-07-01 16:48:27,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: writing mira0435601-115 from 1179648 to 1622016 tid 3 2014-07-01 16:48:27,604.604 INFO:teuthology.orchestra.run.mira043.stdout: building 'simplejson._speedups' extension 2014-07-01 16:48:27,604.604 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.7 -c simplejson/_speedups.c -o build/temp.linux-x86_64-2.7/simplejson/_speedups.o 2014-07-01 16:48:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: writing mira0435601-115 from 1622016 to 1695744 tid 4 2014-07-01 16:48:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 423 current snap is 0 2014-07-01 16:48:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: seq_num 1357 ranges {6057984=12288} 2014-07-01 16:48:27,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: writing mira0435601-423 from 6057984 to 6070272 tid 1 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: read oid 294 snap -1 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 6 to mira0435601-153 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 5 to mira0435601-317 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: finishing write tid 7 to mira0435601-153 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 145 (ObjNum 1351 snap 0 seq_num 1351) dirty exists 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 6 to mira0435601-317 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 201 (ObjNum 1350 snap 0 seq_num 1350) dirty exists 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: finishing write tid 1 to mira0435601-128 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: finishing write tid 3 to mira0435601-128 2014-07-01 16:48:27,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: finishing write tid 4 to mira0435601-128 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 184 (ObjNum 1354 snap 0 seq_num 1354) dirty exists 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: done (6 left) 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: done (5 left) 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: done (4 left) 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2072: read oid 32 snap -1 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 165 current snap is 0 2014-07-01 16:48:27,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: seq_num 1358 ranges {3821568=684032,4505600=450560,4956160=544768,5500928=442368,5943296=167936} 2014-07-01 16:48:27,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: writing mira0435601-165 from 3821568 to 4505600 tid 1 2014-07-01 16:48:27,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: writing mira0435601-165 from 4505600 to 4956160 tid 2 2014-07-01 16:48:27,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: writing mira0435601-165 from 4956160 to 5500928 tid 3 2014-07-01 16:48:27,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: writing mira0435601-165 from 5500928 to 5943296 tid 4 2014-07-01 16:48:27,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: writing mira0435601-165 from 5943296 to 6111232 tid 5 2014-07-01 16:48:27,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: read oid 27 snap -1 2014-07-01 16:48:27,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: expect (ObjNum 714 snap 0 seq_num 714) 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 1 to mira0435601-115 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 1 to mira0435601-165 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 2 to mira0435601-165 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 2 to mira0435601-115 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: finishing write tid 1 to mira0435601-423 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 3 to mira0435601-115 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: finishing write tid 3 to mira0435601-423 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 3 to mira0435601-165 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 4 to mira0435601-165 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 5 to mira0435601-165 2014-07-01 16:48:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: done (6 left) 2014-07-01 16:48:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2075: read oid 387 snap -1 2014-07-01 16:48:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 7 to mira0435601-165 2014-07-01 16:48:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: finishing write tid 8 to mira0435601-165 2014-07-01 16:48:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 223 (ObjNum 1358 snap 0 seq_num 1358) dirty exists 2014-07-01 16:48:27,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: finishing write tid 1 to mira0435601-469 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: finishing write tid 3 to mira0435601-469 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: finishing write tid 4 to mira0435601-469 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 112 (ObjNum 1355 snap 0 seq_num 1355) dirty exists 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 4 to mira0435601-115 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 6 to mira0435601-115 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: finishing write tid 7 to mira0435601-115 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 127 (ObjNum 1356 snap 0 seq_num 1356) dirty exists 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: finishing write tid 4 to mira0435601-423 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 203 (ObjNum 1357 snap 0 seq_num 1357) dirty exists 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: done (6 left) 2014-07-01 16:48:27,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: done (5 left) 2014-07-01 16:48:27,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: done (4 left) 2014-07-01 16:48:27,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2072: done (3 left) 2014-07-01 16:48:27,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: done (2 left) 2014-07-01 16:48:27,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2076: delete oid 285 current snap is 0 2014-07-01 16:48:28,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: expect (ObjNum 1293 snap 0 seq_num 1293) 2014-07-01 16:48:28,516.516 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro build/temp.linux-x86_64-2.7/simplejson/_speedups.o -o build/lib.linux-x86_64-2.7/simplejson/_speedups.so 2014-07-01 16:48:28,548.548 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:28,599.599 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:28,636.636 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for WebOb 2014-07-01 16:48:28,800.800 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:28,810.810 INFO:teuthology.orchestra.run.mira043.stdout: warning: no previously-included files matching '*' found under directory 'docs/_build/_sources' 2014-07-01 16:48:28,836.836 INFO:teuthology.orchestra.run.mira043.stdout: Running setup.py install for greenlet 2014-07-01 16:48:28,939.939 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -fno-tree-dominator-opts -I/usr/include/python2.7 -c /tmp/tmpZ2jc4M/simple.c -o /tmp/tmpZ2jc4M/tmp/tmpZ2jc4M/simple.o 2014-07-01 16:48:28,947.947 INFO:teuthology.orchestra.run.mira043.stdout: /tmp/tmpZ2jc4M/simple.c:1:6: warning: function declaration isn’t a prototype [-Wstrict-prototypes] 2014-07-01 16:48:28,967.967 INFO:teuthology.orchestra.run.mira043.stdout: building 'greenlet' extension 2014-07-01 16:48:28,967.967 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -fno-tree-dominator-opts -I/usr/include/python2.7 -c greenlet.c -o build/temp.linux-x86_64-2.7/greenlet.o 2014-07-01 16:48:29,310.310 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:29,351.351 INFO:teuthology.orchestra.run.mira043.stdout: gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro build/temp.linux-x86_64-2.7/greenlet.o -o build/lib.linux-x86_64-2.7/greenlet.so 2014-07-01 16:48:29,371.371 INFO:teuthology.orchestra.run.mira043.stdout: Linking /home/ubuntu/cephtest/swift/virtualenv/build/greenlet/build/lib.linux-x86_64-2.7/greenlet.so to /home/ubuntu/cephtest/swift/virtualenv/build/greenlet/greenlet.so 2014-07-01 16:48:29,391.391 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:29,432.432 INFO:teuthology.orchestra.run.mira043.stdout:Successfully installed PyYAML nose boto bunch gevent isodate eventlet netifaces xattr simplejson WebOb greenlet 2014-07-01 16:48:29,433.433 INFO:teuthology.orchestra.run.mira043.stdout:Cleaning up... 2014-07-01 16:48:29,866.866 INFO:teuthology.orchestra.run.mira043.stdout:running develop 2014-07-01 16:48:29,878.878 INFO:teuthology.orchestra.run.mira043.stdout:running egg_info 2014-07-01 16:48:29,878.878 INFO:teuthology.orchestra.run.mira043.stdout:creating swift.egg-info 2014-07-01 16:48:29,879.879 INFO:teuthology.orchestra.run.mira043.stdout:writing swift.egg-info/PKG-INFO 2014-07-01 16:48:29,880.880 INFO:teuthology.orchestra.run.mira043.stdout:writing top-level names to swift.egg-info/top_level.txt 2014-07-01 16:48:30,047.047 INFO:teuthology.orchestra.run.mira043.stdout:writing dependency_links to swift.egg-info/dependency_links.txt 2014-07-01 16:48:30,047.047 INFO:teuthology.orchestra.run.mira043.stdout:writing entry points to swift.egg-info/entry_points.txt 2014-07-01 16:48:30,047.047 INFO:teuthology.orchestra.run.mira043.stdout:writing manifest file 'swift.egg-info/SOURCES.txt' 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:reading manifest file 'swift.egg-info/SOURCES.txt' 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:reading manifest template 'MANIFEST.in' 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:writing manifest file 'swift.egg-info/SOURCES.txt' 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:running build_ext 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:Creating /home/ubuntu/cephtest/swift/virtualenv/lib/python2.7/site-packages/swift.egg-link (link to .) 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:Adding swift 1.4.4 to easy-install.pth file 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-account-auditor script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,048.048 INFO:teuthology.orchestra.run.mira043.stderr:warning: no files found matching 'ChangeLog' 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-account-audit script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-account-reaper script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-account-replicator script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-account-server script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-container-auditor script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-container-replicator script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-container-sync script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-container-server script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-container-updater script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-drive-audit script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,049.049 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-get-nodes script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,050.050 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-init script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,050.050 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-auditor script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,050.050 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-expirer script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,050.050 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-info script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,050.050 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-replicator script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,051.051 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-server script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,051.051 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-object-updater script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,051.051 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-proxy-server script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,051.051 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-ring-builder script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,051.051 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-stats-populate script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-stats-report script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-dispersion-populate script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-dispersion-report script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-bench script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-recon script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installing swift-recon-cron script to /home/ubuntu/cephtest/swift/virtualenv/bin 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout: 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Installed /home/ubuntu/cephtest/swift 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Processing dependencies for swift==1.4.4 2014-07-01 16:48:30,052.052 INFO:teuthology.orchestra.run.mira043.stdout:Finished processing dependencies for swift==1.4.4 2014-07-01 16:48:30,053.053 INFO:teuthology.orchestra.run.mira043:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/archive/testswift.client.0.conf' 2014-07-01 16:48:30,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2075: expect (ObjNum 386 snap 0 seq_num 386) 2014-07-01 16:48:30,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: done (2 left) 2014-07-01 16:48:30,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2075: done (1 left) 2014-07-01 16:48:30,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2076: done (0 left) 2014-07-01 16:48:30,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 9 current snap is 0 2014-07-01 16:48:30,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: seq_num 1360 ranges {4964352=585728,5550080=401408} 2014-07-01 16:48:30,873.873 INFO:teuthology.orchestra.run.mira043:Running: "SWIFT_TEST_CONFIG_FILE=/home/ubuntu/cephtest/archive/testswift.client.0.conf /home/ubuntu/cephtest/swift/virtualenv/bin/nosetests -w /home/ubuntu/cephtest/swift/test/functional -v -a '!fails_on_rgw'" 2014-07-01 16:48:30,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: writing mira0435601-9 from 4964352 to 5550080 tid 1 2014-07-01 16:48:30,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: writing mira0435601-9 from 5550080 to 5951488 tid 2 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 410 current snap is 0 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: seq_num 1361 ranges {1179648=454656,1634304=401408} 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: writing mira0435601-410 from 1179648 to 1634304 tid 1 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: writing mira0435601-410 from 1634304 to 2035712 tid 2 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 87 current snap is 0 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: seq_num 1362 ranges {143360=532480} 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: writing mira0435601-87 from 143360 to 675840 tid 1 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: seq_num 1363 ranges {6176768=405504} 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: writing mira0435601-400 from 6176768 to 6582272 tid 1 2014-07-01 16:48:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2081: read oid 50 snap -1 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: finishing write tid 1 to mira0435601-9 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: finishing write tid 2 to mira0435601-9 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: finishing write tid 4 to mira0435601-9 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: finishing write tid 5 to mira0435601-9 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 320 (ObjNum 1360 snap 0 seq_num 1360) dirty exists 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: done (4 left) 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 459 current snap is 0 2014-07-01 16:48:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: seq_num 1364 ranges {6594560=274432} 2014-07-01 16:48:31,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: writing mira0435601-459 from 6594560 to 6868992 tid 1 2014-07-01 16:48:31,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: read oid 329 snap -1 2014-07-01 16:48:31,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 36 current snap is 0 2014-07-01 16:48:31,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: seq_num 1365 ranges {2252800=704512,2957312=585728,3543040=217088} 2014-07-01 16:48:31,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: writing mira0435601-36 from 2252800 to 2957312 tid 1 2014-07-01 16:48:31,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: writing mira0435601-36 from 2957312 to 3543040 tid 2 2014-07-01 16:48:31,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: writing mira0435601-36 from 3543040 to 3760128 tid 3 2014-07-01 16:48:31,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: delete oid 1 current snap is 0 2014-07-01 16:48:31,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: finishing write tid 1 to mira0435601-87 2014-07-01 16:48:31,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: finishing write tid 3 to mira0435601-87 2014-07-01 16:48:31,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: finishing write tid 4 to mira0435601-87 2014-07-01 16:48:31,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 87 v 186 (ObjNum 1362 snap 0 seq_num 1362) dirty exists 2014-07-01 16:48:31,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:2081: expect (ObjNum 1200 snap 0 seq_num 1200) 2014-07-01 16:48:31,731.731 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:32,438.438 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:48:11.321076', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886628372, u'health': u'HEALTH_OK', u'kb_used': 10152680, u'store_stats': {u'bytes_total': 6336854, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'32 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1275/12384 objects degraded (10.296%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 4175383, u'degraded_objects': 1275, u'num_pgs': 364, u'recovering_keys_per_sec': 0, u'data_bytes': 24101492157, u'degraded_total': 12384, u'bytes_used': 47253831680, u'recovering_objects_per_sec': 0, u'write_bytes_sec': 1665668, u'op_per_sec': 12, u'version': 1539, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 332, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1529, u'degraded_ratio': u'10.296', u'bytes_avail': 4951328706560}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 619, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:32,439.439 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:33,209.209 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:48:33,210.210 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:34,029.029 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:34,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: finishing write tid 1 to mira0435601-400 2014-07-01 16:48:34,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: finishing write tid 3 to mira0435601-400 2014-07-01 16:48:34,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 1 to mira0435601-410 2014-07-01 16:48:34,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 2 to mira0435601-410 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 4 to mira0435601-410 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: finishing write tid 1 to mira0435601-459 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: finishing write tid 3 to mira0435601-459 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 5 to mira0435601-410 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 303 (ObjNum 1361 snap 0 seq_num 1361) dirty exists 2014-07-01 16:48:34,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: finishing write tid 1 to mira0435601-36 2014-07-01 16:48:34,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: finishing write tid 2 to mira0435601-36 2014-07-01 16:48:34,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: finishing write tid 3 to mira0435601-36 2014-07-01 16:48:34,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: finishing write tid 5 to mira0435601-36 2014-07-01 16:48:34,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: finishing write tid 6 to mira0435601-36 2014-07-01 16:48:34,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 36 v 307 (ObjNum 1365 snap 0 seq_num 1365) dirty exists 2014-07-01 16:48:34,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: finishing write tid 4 to mira0435601-400 2014-07-01 16:48:34,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 232 (ObjNum 1363 snap 0 seq_num 1363) dirty exists 2014-07-01 16:48:38,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: expect (ObjNum 1057 snap 0 seq_num 1057) 2014-07-01 16:48:39,402.402 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:39,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: finishing write tid 4 to mira0435601-459 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 257 (ObjNum 1364 snap 0 seq_num 1364) dirty exists 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: done (7 left) 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: done (6 left) 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: done (5 left) 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2081: done (4 left) 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: done (3 left) 2014-07-01 16:48:39,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: done (2 left) 2014-07-01 16:48:39,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: done (1 left) 2014-07-01 16:48:39,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: done (0 left) 2014-07-01 16:48:39,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:2086: read oid 485 snap -1 2014-07-01 16:48:39,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2087: read oid 51 snap -1 2014-07-01 16:48:39,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 16:48:39,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: seq_num 1367 ranges {3293184=651264,3944448=401408} 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: writing mira0435601-434 from 3293184 to 3944448 tid 1 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: writing mira0435601-434 from 3944448 to 4345856 tid 2 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2089: read oid 294 snap -1 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2086: done (3 left) 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 436 current snap is 0 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: seq_num 1368 ranges {2387968=524288,2912256=401408} 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: writing mira0435601-436 from 2387968 to 2912256 tid 1 2014-07-01 16:48:39,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: writing mira0435601-436 from 2912256 to 3313664 tid 2 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 324 current snap is 0 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: seq_num 1369 ranges {2195456=794624} 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: writing mira0435601-324 from 2195456 to 2990080 tid 1 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: read oid 476 snap -1 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: seq_num 1370 ranges {45056=421888,466944=638976,1105920=733184,1839104=233472} 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0435601-124 from 45056 to 466944 tid 1 2014-07-01 16:48:39,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0435601-124 from 466944 to 1105920 tid 2 2014-07-01 16:48:39,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0435601-124 from 1105920 to 1839104 tid 3 2014-07-01 16:48:39,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0435601-124 from 1839104 to 2072576 tid 4 2014-07-01 16:48:39,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: read oid 29 snap -1 2014-07-01 16:48:39,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:2089: expect (ObjNum 714 snap 0 seq_num 714) 2014-07-01 16:48:39,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: finishing write tid 1 to mira0435601-434 2014-07-01 16:48:39,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: finishing write tid 2 to mira0435601-434 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: finishing write tid 4 to mira0435601-434 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: finishing write tid 5 to mira0435601-434 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 187 (ObjNum 1367 snap 0 seq_num 1367) dirty exists 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 1 to mira0435601-324 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 3 to mira0435601-324 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 4 to mira0435601-324 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 225 (ObjNum 1369 snap 0 seq_num 1369) dirty exists 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: finishing write tid 1 to mira0435601-436 2014-07-01 16:48:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2087: expect (ObjNum 50 snap 0 seq_num 50) 2014-07-01 16:48:40,003.003 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:40,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: finishing write tid 2 to mira0435601-436 2014-07-01 16:48:40,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: finishing write tid 4 to mira0435601-436 2014-07-01 16:48:40,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 1 to mira0435601-124 2014-07-01 16:48:40,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 2 to mira0435601-124 2014-07-01 16:48:40,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 3 to mira0435601-124 2014-07-01 16:48:40,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 4 to mira0435601-124 2014-07-01 16:48:40,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 6 to mira0435601-124 2014-07-01 16:48:40,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2087: done (7 left) 2014-07-01 16:48:40,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: done (6 left) 2014-07-01 16:48:40,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2089: done (5 left) 2014-07-01 16:48:40,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: done (4 left) 2014-07-01 16:48:40,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: done (3 left) 2014-07-01 16:48:40,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 420 current snap is 0 2014-07-01 16:48:40,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: seq_num 1371 ranges {3956736=344064} 2014-07-01 16:48:40,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: writing mira0435601-420 from 3956736 to 4300800 tid 1 2014-07-01 16:48:40,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 349 current snap is 0 2014-07-01 16:48:40,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: seq_num 1372 ranges {286720=212992} 2014-07-01 16:48:41,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: writing mira0435601-349 from 286720 to 499712 tid 1 2014-07-01 16:48:41,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 35 current snap is 0 2014-07-01 16:48:41,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: seq_num 1373 ranges {1024000=643072,1667072=790528,2457600=12288} 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: writing mira0435601-35 from 1024000 to 1667072 tid 1 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: writing mira0435601-35 from 1667072 to 2457600 tid 2 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: writing mira0435601-35 from 2457600 to 2469888 tid 3 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: read oid 198 snap -1 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: finishing write tid 5 to mira0435601-436 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 184 (ObjNum 1368 snap 0 seq_num 1368) dirty exists 2014-07-01 16:48:41,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: expect (ObjNum 622 snap 0 seq_num 622) 2014-07-01 16:48:42,773.773 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 7 to mira0435601-124 2014-07-01 16:48:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 233 (ObjNum 1370 snap 0 seq_num 1370) dirty exists 2014-07-01 16:48:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: finishing write tid 1 to mira0435601-420 2014-07-01 16:48:43,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: finishing write tid 3 to mira0435601-420 2014-07-01 16:48:43,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 1 to mira0435601-349 2014-07-01 16:48:43,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 3 to mira0435601-349 2014-07-01 16:48:43,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: finishing write tid 1 to mira0435601-35 2014-07-01 16:48:43,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: finishing write tid 2 to mira0435601-35 2014-07-01 16:48:43,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 4 to mira0435601-349 2014-07-01 16:48:43,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 97 (ObjNum 1372 snap 0 seq_num 1372) dirty exists 2014-07-01 16:48:43,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: expect (ObjNum 1189 snap 0 seq_num 1189) 2014-07-01 16:48:43,519.519 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:47:44.712908', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882954044, u'health': u'HEALTH_OK', u'kb_used': 13827008, u'store_stats': {u'bytes_total': 6005101, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907933}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:48:11.321076', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886628372, u'health': u'HEALTH_OK', u'kb_used': 10152680, u'store_stats': {u'bytes_total': 6336854, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs peering'}, {u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'11 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'43 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1236/12301 objects degraded (10.048%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 1236, u'num_pgs': 380, u'data_bytes': 23925126950, u'degraded_total': 12301, u'bytes_used': 46770790400, u'version': 1547, u'pgs_by_state': [{u'count': 10, u'state_name': u'creating'}, {u'count': 1, u'state_name': u'peering'}, {u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 337, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'degraded_ratio': u'10.048', u'bytes_avail': 4951811747840}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 623, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:43,520.520 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:44,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: finishing write tid 3 to mira0435601-35 2014-07-01 16:48:44,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: finishing write tid 5 to mira0435601-35 2014-07-01 16:48:44,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: finishing write tid 4 to mira0435601-420 2014-07-01 16:48:44,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 259 (ObjNum 1371 snap 0 seq_num 1371) dirty exists 2014-07-01 16:48:44,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: finishing write tid 6 to mira0435601-35 2014-07-01 16:48:44,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 237 (ObjNum 1373 snap 0 seq_num 1373) dirty exists 2014-07-01 16:48:44,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: done (6 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: done (5 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: done (4 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: done (3 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: done (2 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: done (1 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: done (0 left) 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2099: read oid 156 snap -1 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 467 current snap is 0 2014-07-01 16:48:44,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: seq_num 1374 ranges {0=516096,516096=622592,1138688=180224} 2014-07-01 16:48:44,305.305 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:48:44,305.305 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: writing mira0435601-467 from 0 to 516096 tid 1 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: writing mira0435601-467 from 516096 to 1138688 tid 2 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: writing mira0435601-467 from 1138688 to 1318912 tid 3 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: read oid 230 snap -1 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2099: done (2 left) 2014-07-01 16:48:44,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: read oid 23 snap -1 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2103: read oid 349 snap -1 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: read oid 392 snap -1 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 364 current snap is 0 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: seq_num 1375 ranges {491520=434176,925696=606208,1531904=450560,1982464=524288,2506752=24576} 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0435601-364 from 491520 to 925696 tid 1 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0435601-364 from 925696 to 1531904 tid 2 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0435601-364 from 1531904 to 1982464 tid 3 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0435601-364 from 1982464 to 2506752 tid 4 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0435601-364 from 2506752 to 2531328 tid 5 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: seq_num 1376 ranges {5296128=462848,5758976=401408} 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: writing mira0435601-409 from 5296128 to 5758976 tid 1 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: writing mira0435601-409 from 5758976 to 6160384 tid 2 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 16:48:44,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: seq_num 1377 ranges {7905280=737280} 2014-07-01 16:48:44,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: writing mira0435601-94 from 7905280 to 8642560 tid 1 2014-07-01 16:48:44,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 16:48:44,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: seq_num 1378 ranges {1789952=606208} 2014-07-01 16:48:44,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: writing mira0435601-444 from 1789952 to 2396160 tid 1 2014-07-01 16:48:44,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2109: read oid 239 snap -1 2014-07-01 16:48:44,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2110: read oid 204 snap -1 2014-07-01 16:48:44,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2103: expect (ObjNum 1372 snap 0 seq_num 1372) 2014-07-01 16:48:44,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: expect (ObjNum 1032 snap 0 seq_num 1032) 2014-07-01 16:48:45,113.113 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:45,153.153 INFO:teuthology.orchestra.run.mira043.stderr:testAccountHead (test.functional.tests.TestAccount) ... ok 2014-07-01 16:48:45,282.282 INFO:teuthology.orchestra.run.mira043.stderr:testContainerListing (test.functional.tests.TestAccount) ... ok 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: finishing write tid 1 to mira0435601-444 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: finishing write tid 3 to mira0435601-444 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: finishing write tid 4 to mira0435601-444 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 189 (ObjNum 1378 snap 0 seq_num 1378) dirty exists 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: finishing write tid 1 to mira0435601-467 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: finishing write tid 2 to mira0435601-467 2014-07-01 16:48:46,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: expect (ObjNum 22 snap 0 seq_num 22) 2014-07-01 16:48:47,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: finishing write tid 3 to mira0435601-467 2014-07-01 16:48:47,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: expect (ObjNum 1260 snap 0 seq_num 1260) 2014-07-01 16:48:48,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: finishing write tid 5 to mira0435601-467 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: finishing write tid 1 to mira0435601-409 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: finishing write tid 2 to mira0435601-409 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: finishing write tid 4 to mira0435601-409 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: finishing write tid 6 to mira0435601-467 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 134 (ObjNum 1374 snap 0 seq_num 1374) dirty exists 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 1 to mira0435601-364 2014-07-01 16:48:48,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: done (10 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: done (9 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: done (8 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2103: done (7 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: done (6 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: done (5 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2109: done (4 left) 2014-07-01 16:48:48,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 259 current snap is 0 2014-07-01 16:48:48,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: seq_num 1379 ranges {0=634880,634880=430080,1064960=450560,1515520=323584} 2014-07-01 16:48:48,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: writing mira0435601-259 from 0 to 634880 tid 1 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: writing mira0435601-259 from 634880 to 1064960 tid 2 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: writing mira0435601-259 from 1064960 to 1515520 tid 3 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: writing mira0435601-259 from 1515520 to 1839104 tid 4 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: delete oid 85 current snap is 0 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 1 to mira0435601-94 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 2 to mira0435601-364 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 3 to mira0435601-94 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 3 to mira0435601-364 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 4 to mira0435601-364 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 5 to mira0435601-364 2014-07-01 16:48:48,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2110: expect (ObjNum 930 snap 0 seq_num 930) 2014-07-01 16:48:50,511.511 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:50,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 7 to mira0435601-364 2014-07-01 16:48:50,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 8 to mira0435601-364 2014-07-01 16:48:50,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 364 v 151 (ObjNum 1375 snap 0 seq_num 1375) dirty exists 2014-07-01 16:48:50,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: finishing write tid 5 to mira0435601-409 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 262 (ObjNum 1376 snap 0 seq_num 1376) dirty exists 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 1 to mira0435601-259 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 4 to mira0435601-94 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 205 (ObjNum 1377 snap 0 seq_num 1377) dirty exists 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 2 to mira0435601-259 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 3 to mira0435601-259 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 4 to mira0435601-259 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 6 to mira0435601-259 2014-07-01 16:48:50,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: done (5 left) 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: done (4 left) 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: done (3 left) 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2110: done (2 left) 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: done (1 left) 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: read oid 463 snap -1 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: finishing write tid 7 to mira0435601-259 2014-07-01 16:48:50,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 235 (ObjNum 1379 snap 0 seq_num 1379) dirty exists 2014-07-01 16:48:50,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: done (1 left) 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: seq_num 1381 ranges {2027520=24576} 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: writing mira0435601-336 from 2027520 to 2052096 tid 1 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 23 current snap is 0 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: seq_num 1382 ranges {1794048=454656,2248704=425984,2674688=376832} 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0435601-23 from 1794048 to 2248704 tid 1 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0435601-23 from 2248704 to 2674688 tid 2 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0435601-23 from 2674688 to 3051520 tid 3 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: read oid 240 snap -1 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: read oid 68 snap -1 2014-07-01 16:48:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: expect (ObjNum 1259 snap 0 seq_num 1259) 2014-07-01 16:48:51,282.282 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:51,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 1 to mira0435601-336 2014-07-01 16:48:51,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 3 to mira0435601-336 2014-07-01 16:48:51,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: expect (ObjNum 1143 snap 0 seq_num 1143) 2014-07-01 16:48:52,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 4 to mira0435601-336 2014-07-01 16:48:52,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 146 (ObjNum 1381 snap 0 seq_num 1381) dirty exists 2014-07-01 16:48:52,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: expect (ObjNum 924 snap 0 seq_num 924) 2014-07-01 16:48:53,861.861 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:54,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 1 to mira0435601-23 2014-07-01 16:48:54,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 2 to mira0435601-23 2014-07-01 16:48:54,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 3 to mira0435601-23 2014-07-01 16:48:54,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 5 to mira0435601-23 2014-07-01 16:48:54,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 6 to mira0435601-23 2014-07-01 16:48:54,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 155 (ObjNum 1382 snap 0 seq_num 1382) dirty exists 2014-07-01 16:48:54,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: done (4 left) 2014-07-01 16:48:54,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: done (3 left) 2014-07-01 16:48:54,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: done (2 left) 2014-07-01 16:48:54,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: done (1 left) 2014-07-01 16:48:54,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: done (0 left) 2014-07-01 16:48:54,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 114 current snap is 0 2014-07-01 16:48:54,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: seq_num 1383 ranges {2740224=81920} 2014-07-01 16:48:54,611.611 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:48:44.721239', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882876592, u'health': u'HEALTH_OK', u'kb_used': 13904460, u'store_stats': {u'bytes_total': 7386634, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4240890}}, {u'last_updated': u'2014-07-01 07:47:54.538147', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886637016, u'health': u'HEALTH_OK', u'kb_used': 10144036, u'store_stats': {u'bytes_total': 7053042, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3907298}}, {u'last_updated': u'2014-07-01 07:48:11.321076', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886628372, u'health': u'HEALTH_OK', u'kb_used': 10152680, u'store_stats': {u'bytes_total': 6336854, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'13 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'8 pgs stuck inactive'}, {u'severity': u'HEALTH_WARN', u'summary': u'40 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1143/12055 objects degraded (9.482%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 1143, u'num_pgs': 388, u'data_bytes': 23248444815, u'degraded_total': 12055, u'bytes_used': 45574807552, u'op_per_sec': 18, u'write_bytes_sec': 31450, u'version': 1555, u'pgs_by_state': [{u'count': 8, u'state_name': u'creating'}, {u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 348, u'state_name': u'active+clean'}, {u'count': 13, u'state_name': u'active+recovering'}], u'degraded_ratio': u'9.482', u'bytes_avail': 4953007730688}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 625, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:48:54,611.611 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:48:54,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: writing mira0435601-114 from 2740224 to 2822144 tid 1 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 101 current snap is 0 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: seq_num 1384 ranges {0=647168,647168=643072,1290240=561152} 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: writing mira0435601-101 from 0 to 647168 tid 1 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: writing mira0435601-101 from 647168 to 1290240 tid 2 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: writing mira0435601-101 from 1290240 to 1851392 tid 3 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: delete oid 249 current snap is 0 2014-07-01 16:48:54,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 1 to mira0435601-114 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 3 to mira0435601-114 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 4 to mira0435601-114 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 191 (ObjNum 1383 snap 0 seq_num 1383) dirty exists 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: done (2 left) 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: done (1 left) 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2121: read oid 373 snap -1 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 187 current snap is 0 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: seq_num 1386 ranges {1888256=704512,2592768=405504,2998272=671744,3670016=126976} 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: writing mira0435601-187 from 1888256 to 2592768 tid 1 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: writing mira0435601-187 from 2592768 to 2998272 tid 2 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: writing mira0435601-187 from 2998272 to 3670016 tid 3 2014-07-01 16:48:54,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: writing mira0435601-187 from 3670016 to 3796992 tid 4 2014-07-01 16:48:54,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 136 current snap is 0 2014-07-01 16:48:54,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: seq_num 1387 ranges {0=417792} 2014-07-01 16:48:54,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: writing mira0435601-136 from 0 to 417792 tid 1 2014-07-01 16:48:54,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: delete oid 186 current snap is 0 2014-07-01 16:48:54,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2121: expect (ObjNum 698 snap 0 seq_num 698) 2014-07-01 16:48:55,395.395 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:48:55,395.395 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:48:56,234.234 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:48:57,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: finishing write tid 1 to mira0435601-136 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: finishing write tid 3 to mira0435601-136 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: finishing write tid 4 to mira0435601-136 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 250 (ObjNum 1387 snap 0 seq_num 1387) dirty exists 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: finishing write tid 1 to mira0435601-101 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2121: done (4 left) 2014-07-01 16:48:57,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: done (3 left) 2014-07-01 16:48:57,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: done (2 left) 2014-07-01 16:48:57,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2125: delete oid 141 current snap is 0 2014-07-01 16:48:57,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: finishing write tid 2 to mira0435601-101 2014-07-01 16:48:57,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: finishing write tid 3 to mira0435601-101 2014-07-01 16:48:57,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: finishing write tid 5 to mira0435601-101 2014-07-01 16:48:57,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: finishing write tid 6 to mira0435601-101 2014-07-01 16:48:57,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 125 (ObjNum 1384 snap 0 seq_num 1384) dirty exists 2014-07-01 16:48:57,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 1 to mira0435601-187 2014-07-01 16:48:57,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 2 to mira0435601-187 2014-07-01 16:48:57,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 3 to mira0435601-187 2014-07-01 16:48:57,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 4 to mira0435601-187 2014-07-01 16:48:57,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 6 to mira0435601-187 2014-07-01 16:48:57,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: finishing write tid 7 to mira0435601-187 2014-07-01 16:48:57,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 240 (ObjNum 1386 snap 0 seq_num 1386) dirty exists 2014-07-01 16:48:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: done (2 left) 2014-07-01 16:48:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: done (1 left) 2014-07-01 16:48:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2125: done (0 left) 2014-07-01 16:48:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: read oid 162 snap -1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2127: read oid 465 snap -1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: read oid 130 snap -1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 50 current snap is 0 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: seq_num 1390 ranges {4739072=638976,5378048=634880,6012928=409600,6422528=548864,6971392=12288} 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: writing mira0435601-50 from 4739072 to 5378048 tid 1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: writing mira0435601-50 from 5378048 to 6012928 tid 2 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: writing mira0435601-50 from 6012928 to 6422528 tid 3 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: writing mira0435601-50 from 6422528 to 6971392 tid 4 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: writing mira0435601-50 from 6971392 to 6983680 tid 5 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: seq_num 1391 ranges {0=667648,667648=401408} 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: writing mira0435601-382 from 0 to 667648 tid 1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: writing mira0435601-382 from 667648 to 1069056 tid 2 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: read oid 207 snap -1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2127: done (5 left) 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: read oid 201 snap -1 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 78 current snap is 0 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: seq_num 1392 ranges {335872=692224,1028096=446464,1474560=512000,1986560=647168} 2014-07-01 16:48:57,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: writing mira0435601-78 from 335872 to 1028096 tid 1 2014-07-01 16:48:57,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: writing mira0435601-78 from 1028096 to 1474560 tid 2 2014-07-01 16:48:57,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: writing mira0435601-78 from 1474560 to 1986560 tid 3 2014-07-01 16:48:57,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: writing mira0435601-78 from 1986560 to 2633728 tid 4 2014-07-01 16:48:57,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:2134: read oid 45 snap -1 2014-07-01 16:48:57,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: delete oid 357 current snap is 0 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: done (8 left) 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 262 current snap is 0 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: seq_num 1394 ranges {4976640=487424} 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: writing mira0435601-262 from 4976640 to 5464064 tid 1 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 146 current snap is 0 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: seq_num 1395 ranges {1400832=356352} 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: writing mira0435601-146 from 1400832 to 1757184 tid 1 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 483 current snap is 0 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: seq_num 1396 ranges {1855488=786432,2641920=430080,3072000=372736} 2014-07-01 16:48:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: writing mira0435601-483 from 1855488 to 2641920 tid 1 2014-07-01 16:48:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: writing mira0435601-483 from 2641920 to 3072000 tid 2 2014-07-01 16:48:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: writing mira0435601-483 from 3072000 to 3444736 tid 3 2014-07-01 16:48:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 209 current snap is 0 2014-07-01 16:48:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: seq_num 1397 ranges {1970176=659456,2629632=765952,3395584=36864} 2014-07-01 16:48:57,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: writing mira0435601-209 from 1970176 to 2629632 tid 1 2014-07-01 16:48:57,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: writing mira0435601-209 from 2629632 to 3395584 tid 2 2014-07-01 16:48:57,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: writing mira0435601-209 from 3395584 to 3432448 tid 3 2014-07-01 16:48:57,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2140: read oid 310 snap -1 2014-07-01 16:48:57,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 1 to mira0435601-50 2014-07-01 16:48:57,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 2 to mira0435601-50 2014-07-01 16:48:57,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 224 current snap is 0 2014-07-01 16:48:57,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: seq_num 1398 ranges {2785280=577536,3362816=593920,3956736=458752,4415488=458752,4874240=94208} 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: writing mira0435601-224 from 2785280 to 3362816 tid 1 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: writing mira0435601-224 from 3362816 to 3956736 tid 2 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: writing mira0435601-224 from 3956736 to 4415488 tid 3 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: writing mira0435601-224 from 4415488 to 4874240 tid 4 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: writing mira0435601-224 from 4874240 to 4968448 tid 5 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2142: read oid 77 snap -1 2014-07-01 16:48:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: expect (ObjNum 1133 snap 0 seq_num 1133) 2014-07-01 16:48:58,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: expect (ObjNum 129 snap 0 seq_num 129) 2014-07-01 16:48:58,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: finishing write tid 1 to mira0435601-382 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 3 to mira0435601-50 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: finishing write tid 2 to mira0435601-382 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 4 to mira0435601-50 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: finishing write tid 4 to mira0435601-382 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 5 to mira0435601-50 2014-07-01 16:48:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: expect (ObjNum 200 snap 0 seq_num 200) 2014-07-01 16:48:59,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2142: expect (ObjNum 76 snap 0 seq_num 76) 2014-07-01 16:49:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 7 to mira0435601-50 2014-07-01 16:49:00,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: finishing write tid 8 to mira0435601-50 2014-07-01 16:49:00,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 189 (ObjNum 1390 snap 0 seq_num 1390) dirty exists 2014-07-01 16:49:00,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: finishing write tid 5 to mira0435601-382 2014-07-01 16:49:00,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 106 (ObjNum 1391 snap 0 seq_num 1391) dirty exists 2014-07-01 16:49:00,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: finishing write tid 1 to mira0435601-262 2014-07-01 16:49:00,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: finishing write tid 3 to mira0435601-262 2014-07-01 16:49:00,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: finishing write tid 1 to mira0435601-209 2014-07-01 16:49:00,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: finishing write tid 2 to mira0435601-209 2014-07-01 16:49:00,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:49:01,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 1 to mira0435601-78 2014-07-01 16:49:01,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 2 to mira0435601-78 2014-07-01 16:49:01,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 3 to mira0435601-78 2014-07-01 16:49:01,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 4 to mira0435601-78 2014-07-01 16:49:01,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: finishing write tid 3 to mira0435601-209 2014-07-01 16:49:01,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 6 to mira0435601-78 2014-07-01 16:49:01,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: finishing write tid 5 to mira0435601-209 2014-07-01 16:49:01,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 7 to mira0435601-78 2014-07-01 16:49:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 189 (ObjNum 1392 snap 0 seq_num 1392) dirty exists 2014-07-01 16:49:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: done (14 left) 2014-07-01 16:49:01,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: done (13 left) 2014-07-01 16:49:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: done (12 left) 2014-07-01 16:49:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: done (11 left) 2014-07-01 16:49:01,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: done (10 left) 2014-07-01 16:49:01,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: done (9 left) 2014-07-01 16:49:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: done (8 left) 2014-07-01 16:49:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2142: done (7 left) 2014-07-01 16:49:01,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 35 current snap is 0 2014-07-01 16:49:01,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: seq_num 1399 ranges {2469888=606208,3076096=401408} 2014-07-01 16:49:01,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: writing mira0435601-35 from 2469888 to 3076096 tid 1 2014-07-01 16:49:01,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: writing mira0435601-35 from 3076096 to 3477504 tid 2 2014-07-01 16:49:01,756.756 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2144: read oid 286 snap -1 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: seq_num 1400 ranges {2678784=479232,3158016=401408} 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: writing mira0435601-273 from 2678784 to 3158016 tid 1 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: writing mira0435601-273 from 3158016 to 3559424 tid 2 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 104 current snap is 0 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: seq_num 1401 ranges {0=749568} 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: writing mira0435601-104 from 0 to 749568 tid 1 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2147: read oid 167 snap -1 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: finishing write tid 1 to mira0435601-146 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: finishing write tid 3 to mira0435601-146 2014-07-01 16:49:01,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2140: expect (ObjNum 1271 snap 0 seq_num 1271) 2014-07-01 16:49:02,529.529 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:04,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 1 to mira0435601-224 2014-07-01 16:49:04,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 2 to mira0435601-224 2014-07-01 16:49:04,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 3 to mira0435601-224 2014-07-01 16:49:04,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2134: expect (ObjNum 1108 snap 0 seq_num 1108) 2014-07-01 16:49:05,489.489 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:06,279.279 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:48:44.721239', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882876592, u'health': u'HEALTH_OK', u'kb_used': 13904460, u'store_stats': {u'bytes_total': 7386634, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4240890}}, {u'last_updated': u'2014-07-01 07:48:54.538491', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886606904, u'health': u'HEALTH_OK', u'kb_used': 10174148, u'store_stats': {u'bytes_total': 9482575, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239679}}, {u'last_updated': u'2014-07-01 07:48:11.321076', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886628372, u'health': u'HEALTH_OK', u'kb_used': 10152680, u'store_stats': {u'bytes_total': 6336854, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 1037/11889 objects degraded (8.722%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 1037, u'num_pgs': 388, u'data_bytes': 22525234933, u'degraded_total': 11889, u'bytes_used': 44433158144, u'op_per_sec': 236, u'write_bytes_sec': 1132670, u'version': 1566, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 358, u'state_name': u'active+clean'}, {u'count': 11, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1483101, u'degraded_ratio': u'8.722', u'bytes_avail': 4954149380096}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:49:06,280.280 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:07,207.207 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:49:07,208.208 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:08,229.229 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:09,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: finishing write tid 4 to mira0435601-146 2014-07-01 16:49:09,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 157 (ObjNum 1395 snap 0 seq_num 1395) dirty exists 2014-07-01 16:49:09,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 4 to mira0435601-224 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 5 to mira0435601-224 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 7 to mira0435601-224 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: finishing write tid 4 to mira0435601-262 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 254 (ObjNum 1394 snap 0 seq_num 1394) dirty exists 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 1 to mira0435601-483 2014-07-01 16:49:09,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 2 to mira0435601-483 2014-07-01 16:49:09,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 3 to mira0435601-483 2014-07-01 16:49:09,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: finishing write tid 6 to mira0435601-209 2014-07-01 16:49:09,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 209 v 132 (ObjNum 1397 snap 0 seq_num 1397) dirty exists 2014-07-01 16:49:09,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 5 to mira0435601-483 2014-07-01 16:49:09,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2144: expect (ObjNum 736 snap 0 seq_num 736) 2014-07-01 16:49:10,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2134: done (11 left) 2014-07-01 16:49:10,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: done (10 left) 2014-07-01 16:49:10,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: done (9 left) 2014-07-01 16:49:10,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: done (8 left) 2014-07-01 16:49:10,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2140: done (7 left) 2014-07-01 16:49:10,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2144: done (6 left) 2014-07-01 16:49:10,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2147: done (5 left) 2014-07-01 16:49:10,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 406 current snap is 0 2014-07-01 16:49:10,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: seq_num 1402 ranges {126976=425984} 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: writing mira0435601-406 from 126976 to 552960 tid 1 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2149: read oid 392 snap -1 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 6 to mira0435601-483 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 150 (ObjNum 1396 snap 0 seq_num 1396) dirty exists 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: finishing write tid 8 to mira0435601-224 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 243 (ObjNum 1398 snap 0 seq_num 1398) dirty exists 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: finishing write tid 1 to mira0435601-35 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: finishing write tid 1 to mira0435601-273 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: finishing write tid 2 to mira0435601-35 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: finishing write tid 2 to mira0435601-273 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: finishing write tid 4 to mira0435601-35 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 1 to mira0435601-104 2014-07-01 16:49:11,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 3 to mira0435601-104 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: finishing write tid 4 to mira0435601-273 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: done (6 left) 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: done (5 left) 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 495 current snap is 0 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: seq_num 1403 ranges {1343488=454656,1798144=405504,2203648=577536,2781184=221184} 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: writing mira0435601-495 from 1343488 to 1798144 tid 1 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: writing mira0435601-495 from 1798144 to 2203648 tid 2 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: writing mira0435601-495 from 2203648 to 2781184 tid 3 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: writing mira0435601-495 from 2781184 to 3002368 tid 4 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 170 current snap is 0 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: seq_num 1404 ranges {4755456=167936} 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: writing mira0435601-170 from 4755456 to 4923392 tid 1 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 422 current snap is 0 2014-07-01 16:49:11,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: seq_num 1405 ranges {241664=598016,839680=622592,1462272=180224} 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0435601-422 from 241664 to 839680 tid 1 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0435601-422 from 839680 to 1462272 tid 2 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0435601-422 from 1462272 to 1642496 tid 3 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2153: read oid 242 snap -1 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: finishing write tid 5 to mira0435601-273 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 153 (ObjNum 1400 snap 0 seq_num 1400) dirty exists 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: done (8 left) 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2154: read oid 253 snap -1 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: finishing write tid 5 to mira0435601-35 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 246 (ObjNum 1399 snap 0 seq_num 1399) dirty exists 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 4 to mira0435601-104 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 104 v 264 (ObjNum 1401 snap 0 seq_num 1401) dirty exists 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: finishing write tid 1 to mira0435601-406 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: finishing write tid 3 to mira0435601-406 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 1 to mira0435601-495 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: finishing write tid 1 to mira0435601-170 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: finishing write tid 3 to mira0435601-170 2014-07-01 16:49:11,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: finishing write tid 4 to mira0435601-170 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 194 (ObjNum 1404 snap 0 seq_num 1404) dirty exists 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: done (8 left) 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: done (7 left) 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: done (6 left) 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2155: read oid 4 snap -1 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: seq_num 1406 ranges {0=225280} 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: writing mira0435601-186 from 0 to 225280 tid 1 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2157: read oid 336 snap -1 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2158: read oid 265 snap -1 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: read oid 22 snap -1 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 432 current snap is 0 2014-07-01 16:49:11,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: seq_num 1407 ranges {3690496=790528,4481024=786432,5267456=417792} 2014-07-01 16:49:11,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: writing mira0435601-432 from 3690496 to 4481024 tid 1 2014-07-01 16:49:11,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: writing mira0435601-432 from 4481024 to 5267456 tid 2 2014-07-01 16:49:11,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: writing mira0435601-432 from 5267456 to 5685248 tid 3 2014-07-01 16:49:11,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 219 current snap is 0 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: seq_num 1408 ranges {1970176=417792,2387968=401408} 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: writing mira0435601-219 from 1970176 to 2387968 tid 1 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: writing mira0435601-219 from 2387968 to 2789376 tid 2 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: seq_num 1409 ranges {2396160=442368,2838528=602112,3440640=524288,3964928=479232} 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0435601-444 from 2396160 to 2838528 tid 1 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0435601-444 from 2838528 to 3440640 tid 2 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0435601-444 from 3440640 to 3964928 tid 3 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0435601-444 from 3964928 to 4444160 tid 4 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2163: read oid 87 snap -1 2014-07-01 16:49:11,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 2 to mira0435601-495 2014-07-01 16:49:11,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2154: expect (ObjNum 804 snap 0 seq_num 804) 2014-07-01 16:49:13,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: finishing write tid 4 to mira0435601-406 2014-07-01 16:49:13,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 406 v 191 (ObjNum 1402 snap 0 seq_num 1402) dirty exists 2014-07-01 16:49:13,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 3 to mira0435601-495 2014-07-01 16:49:13,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 4 to mira0435601-495 2014-07-01 16:49:13,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 6 to mira0435601-495 2014-07-01 16:49:13,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: expect (ObjNum 638 snap 0 seq_num 638) 2014-07-01 16:49:14,288.288 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:15,209.209 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:15,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 1 to mira0435601-444 2014-07-01 16:49:15,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:2149: expect (ObjNum 1260 snap 0 seq_num 1260) 2014-07-01 16:49:16,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 1 to mira0435601-422 2014-07-01 16:49:16,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 2 to mira0435601-422 2014-07-01 16:49:16,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 3 to mira0435601-422 2014-07-01 16:49:16,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 5 to mira0435601-422 2014-07-01 16:49:16,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2153: expect (ObjNum 742 snap 0 seq_num 742) 2014-07-01 16:49:18,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 2 to mira0435601-444 2014-07-01 16:49:18,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 3 to mira0435601-444 2014-07-01 16:49:18,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 4 to mira0435601-444 2014-07-01 16:49:18,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 6 to mira0435601-444 2014-07-01 16:49:18,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 7 to mira0435601-444 2014-07-01 16:49:18,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 199 (ObjNum 1409 snap 0 seq_num 1409) dirty exists 2014-07-01 16:49:18,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 6 to mira0435601-422 2014-07-01 16:49:18,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 195 (ObjNum 1405 snap 0 seq_num 1405) dirty exists 2014-07-01 16:49:18,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:2163: expect (ObjNum 1362 snap 0 seq_num 1362) 2014-07-01 16:49:18,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 1 to mira0435601-186 2014-07-01 16:49:18,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: finishing write tid 7 to mira0435601-495 2014-07-01 16:49:18,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 188 (ObjNum 1403 snap 0 seq_num 1403) dirty exists 2014-07-01 16:49:18,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 3 to mira0435601-186 2014-07-01 16:49:18,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: finishing write tid 1 to mira0435601-432 2014-07-01 16:49:18,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: finishing write tid 2 to mira0435601-432 2014-07-01 16:49:18,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: finishing write tid 3 to mira0435601-432 2014-07-01 16:49:18,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: done (14 left) 2014-07-01 16:49:18,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2149: done (13 left) 2014-07-01 16:49:18,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: done (12 left) 2014-07-01 16:49:18,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: done (11 left) 2014-07-01 16:49:18,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2153: done (10 left) 2014-07-01 16:49:18,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2154: done (9 left) 2014-07-01 16:49:18,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: done (8 left) 2014-07-01 16:49:18,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: done (7 left) 2014-07-01 16:49:18,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2163: done (6 left) 2014-07-01 16:49:18,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2164: read oid 426 snap -1 2014-07-01 16:49:18,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2157: expect (ObjNum 1381 snap 0 seq_num 1381) 2014-07-01 16:49:18,824.824 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:19,594.594 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:48:44.721239', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882876592, u'health': u'HEALTH_OK', u'kb_used': 13904460, u'store_stats': {u'bytes_total': 7386634, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4240890}}, {u'last_updated': u'2014-07-01 07:48:54.538491', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886606904, u'health': u'HEALTH_OK', u'kb_used': 10174148, u'store_stats': {u'bytes_total': 9482575, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239679}}, {u'last_updated': u'2014-07-01 07:49:11.321414', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886593676, u'health': u'HEALTH_OK', u'kb_used': 10187376, u'store_stats': {u'bytes_total': 11579734, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 940/11648 objects degraded (8.070%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 940, u'num_pgs': 388, u'data_bytes': 21956664126, u'degraded_total': 11648, u'bytes_used': 43387748352, u'op_per_sec': 4, u'write_bytes_sec': 0, u'version': 1577, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 358, u'state_name': u'active+clean'}, {u'count': 11, u'state_name': u'active+recovering'}], u'degraded_ratio': u'8.070', u'bytes_avail': 4955194789888}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:49:19,595.595 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:19,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 4 to mira0435601-186 2014-07-01 16:49:19,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 130 (ObjNum 1406 snap 0 seq_num 1406) dirty exists 2014-07-01 16:49:19,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: finishing write tid 5 to mira0435601-432 2014-07-01 16:49:19,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: finishing write tid 1 to mira0435601-219 2014-07-01 16:49:19,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: finishing write tid 6 to mira0435601-432 2014-07-01 16:49:19,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 112 (ObjNum 1407 snap 0 seq_num 1407) dirty exists 2014-07-01 16:49:19,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: finishing write tid 2 to mira0435601-219 2014-07-01 16:49:19,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: finishing write tid 4 to mira0435601-219 2014-07-01 16:49:19,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:2155: expect (ObjNum 955 snap 0 seq_num 955) 2014-07-01 16:49:20,471.471 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:49:20,471.471 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:21,481.481 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:21,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: finishing write tid 5 to mira0435601-219 2014-07-01 16:49:21,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 267 (ObjNum 1408 snap 0 seq_num 1408) dirty exists 2014-07-01 16:49:21,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2155: done (6 left) 2014-07-01 16:49:21,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: done (5 left) 2014-07-01 16:49:21,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2157: done (4 left) 2014-07-01 16:49:21,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: done (3 left) 2014-07-01 16:49:21,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: done (2 left) 2014-07-01 16:49:21,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 474 current snap is 0 2014-07-01 16:49:21,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: seq_num 1410 ranges {3035136=561152} 2014-07-01 16:49:21,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: writing mira0435601-474 from 3035136 to 3596288 tid 1 2014-07-01 16:49:21,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 479 current snap is 0 2014-07-01 16:49:21,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: seq_num 1411 ranges {2060288=237568} 2014-07-01 16:49:21,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: writing mira0435601-479 from 2060288 to 2297856 tid 1 2014-07-01 16:49:21,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: read oid 244 snap -1 2014-07-01 16:49:21,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2158: expect (ObjNum 846 snap 0 seq_num 846) 2014-07-01 16:49:23,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2164: expect (ObjNum 425 snap 0 seq_num 425) 2014-07-01 16:49:24,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 1 to mira0435601-479 2014-07-01 16:49:24,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 3 to mira0435601-479 2014-07-01 16:49:24,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: expect (ObjNum 1150 snap 0 seq_num 1150) 2014-07-01 16:49:25,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2158: done (4 left) 2014-07-01 16:49:25,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:2164: done (3 left) 2014-07-01 16:49:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: done (2 left) 2014-07-01 16:49:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 74 current snap is 0 2014-07-01 16:49:25,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: seq_num 1412 ranges {1044480=663552,1708032=598016,2306048=204800} 2014-07-01 16:49:25,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: writing mira0435601-74 from 1044480 to 1708032 tid 1 2014-07-01 16:49:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: writing mira0435601-74 from 1708032 to 2306048 tid 2 2014-07-01 16:49:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: writing mira0435601-74 from 2306048 to 2510848 tid 3 2014-07-01 16:49:25,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:2169: delete oid 188 current snap is 0 2014-07-01 16:49:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 4 to mira0435601-479 2014-07-01 16:49:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 210 (ObjNum 1411 snap 0 seq_num 1411) dirty exists 2014-07-01 16:49:25,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: finishing write tid 1 to mira0435601-474 2014-07-01 16:49:25,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: finishing write tid 1 to mira0435601-74 2014-07-01 16:49:25,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: finishing write tid 3 to mira0435601-474 2014-07-01 16:49:25,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: finishing write tid 4 to mira0435601-474 2014-07-01 16:49:25,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 159 (ObjNum 1410 snap 0 seq_num 1410) dirty exists 2014-07-01 16:49:25,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: finishing write tid 2 to mira0435601-74 2014-07-01 16:49:25,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: finishing write tid 3 to mira0435601-74 2014-07-01 16:49:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: finishing write tid 5 to mira0435601-74 2014-07-01 16:49:26,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: finishing write tid 6 to mira0435601-74 2014-07-01 16:49:26,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 271 (ObjNum 1412 snap 0 seq_num 1412) dirty exists 2014-07-01 16:49:27,073.073 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:27,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: done (3 left) 2014-07-01 16:49:27,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: done (2 left) 2014-07-01 16:49:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: done (1 left) 2014-07-01 16:49:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2169: done (0 left) 2014-07-01 16:49:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 67 current snap is 0 2014-07-01 16:49:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: seq_num 1414 ranges {2121728=409600,2531328=761856,3293184=40960} 2014-07-01 16:49:27,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: writing mira0435601-67 from 2121728 to 2531328 tid 1 2014-07-01 16:49:27,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: writing mira0435601-67 from 2531328 to 3293184 tid 2 2014-07-01 16:49:27,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: writing mira0435601-67 from 3293184 to 3334144 tid 3 2014-07-01 16:49:27,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 360 current snap is 0 2014-07-01 16:49:27,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: seq_num 1415 ranges {0=32768} 2014-07-01 16:49:27,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: writing mira0435601-360 from 0 to 32768 tid 1 2014-07-01 16:49:27,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2172: read oid 430 snap -1 2014-07-01 16:49:27,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 325 current snap is 0 2014-07-01 16:49:27,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: seq_num 1416 ranges {3772416=757760} 2014-07-01 16:49:27,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: writing mira0435601-325 from 3772416 to 4530176 tid 1 2014-07-01 16:49:27,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 161 current snap is 0 2014-07-01 16:49:27,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: seq_num 1417 ranges {1097728=630784} 2014-07-01 16:49:27,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: writing mira0435601-161 from 1097728 to 1728512 tid 1 2014-07-01 16:49:27,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 282 current snap is 0 2014-07-01 16:49:27,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: seq_num 1418 ranges {2007040=655360,2662400=745472,3407872=458752} 2014-07-01 16:49:27,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: writing mira0435601-282 from 2007040 to 2662400 tid 1 2014-07-01 16:49:27,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: writing mira0435601-282 from 2662400 to 3407872 tid 2 2014-07-01 16:49:27,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: writing mira0435601-282 from 3407872 to 3866624 tid 3 2014-07-01 16:49:27,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2176: read oid 372 snap -1 2014-07-01 16:49:27,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 486 current snap is 0 2014-07-01 16:49:27,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: seq_num 1419 ranges {2330624=528384,2859008=573440,3432448=630784} 2014-07-01 16:49:27,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: writing mira0435601-486 from 2330624 to 2859008 tid 1 2014-07-01 16:49:27,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: writing mira0435601-486 from 2859008 to 3432448 tid 2 2014-07-01 16:49:27,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: writing mira0435601-486 from 3432448 to 4063232 tid 3 2014-07-01 16:49:27,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 424 current snap is 0 2014-07-01 16:49:27,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: seq_num 1420 ranges {2965504=802816,3768320=557056,4325376=245760} 2014-07-01 16:49:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: writing mira0435601-424 from 2965504 to 3768320 tid 1 2014-07-01 16:49:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: writing mira0435601-424 from 3768320 to 4325376 tid 2 2014-07-01 16:49:27,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: writing mira0435601-424 from 4325376 to 4571136 tid 3 2014-07-01 16:49:27,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2179: read oid 121 snap -1 2014-07-01 16:49:27,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2172: expect (ObjNum 690 snap 0 seq_num 690) 2014-07-01 16:49:27,861.861 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: finishing write tid 1 to mira0435601-424 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: finishing write tid 2 to mira0435601-424 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: finishing write tid 3 to mira0435601-424 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: finishing write tid 5 to mira0435601-424 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: finishing write tid 6 to mira0435601-424 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 229 (ObjNum 1420 snap 0 seq_num 1420) dirty exists 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: finishing write tid 1 to mira0435601-67 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: finishing write tid 1 to mira0435601-282 2014-07-01 16:49:29,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2172: done (9 left) 2014-07-01 16:49:29,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2176: done (8 left) 2014-07-01 16:49:29,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: done (7 left) 2014-07-01 16:49:29,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 421 current snap is 0 2014-07-01 16:49:29,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: seq_num 1421 ranges {135168=475136,610304=634880,1245184=167936} 2014-07-01 16:49:29,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: writing mira0435601-421 from 135168 to 610304 tid 1 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: writing mira0435601-421 from 610304 to 1245184 tid 2 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: writing mira0435601-421 from 1245184 to 1413120 tid 3 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: finishing write tid 2 to mira0435601-282 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: finishing write tid 3 to mira0435601-282 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: finishing write tid 5 to mira0435601-282 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 1 to mira0435601-360 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: finishing write tid 2 to mira0435601-67 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 3 to mira0435601-360 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: finishing write tid 3 to mira0435601-67 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: finishing write tid 5 to mira0435601-67 2014-07-01 16:49:30,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: finishing write tid 6 to mira0435601-67 2014-07-01 16:49:30,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 199 (ObjNum 1414 snap 0 seq_num 1414) dirty exists 2014-07-01 16:49:30,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 4 to mira0435601-360 2014-07-01 16:49:30,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 133 (ObjNum 1415 snap 0 seq_num 1415) dirty exists 2014-07-01 16:49:30,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2179: expect (ObjNum 893 snap 0 seq_num 893) 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: finishing write tid 6 to mira0435601-282 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 250 (ObjNum 1418 snap 0 seq_num 1418) dirty exists 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: finishing write tid 1 to mira0435601-325 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: finishing write tid 3 to mira0435601-325 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: finishing write tid 4 to mira0435601-325 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 155 (ObjNum 1416 snap 0 seq_num 1416) dirty exists 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: finishing write tid 1 to mira0435601-486 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: finishing write tid 1 to mira0435601-161 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: finishing write tid 3 to mira0435601-161 2014-07-01 16:49:30,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: finishing write tid 4 to mira0435601-161 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 118 (ObjNum 1417 snap 0 seq_num 1417) dirty exists 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: finishing write tid 1 to mira0435601-421 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: read oid 189 snap -1 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: finishing write tid 2 to mira0435601-421 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: finishing write tid 3 to mira0435601-421 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: finishing write tid 5 to mira0435601-421 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: finishing write tid 6 to mira0435601-421 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 236 (ObjNum 1421 snap 0 seq_num 1421) dirty exists 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: done (8 left) 2014-07-01 16:49:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: done (7 left) 2014-07-01 16:49:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: done (6 left) 2014-07-01 16:49:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: done (5 left) 2014-07-01 16:49:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: done (4 left) 2014-07-01 16:49:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2179: done (3 left) 2014-07-01 16:49:30,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: done (2 left) 2014-07-01 16:49:30,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2182: read oid 191 snap -1 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 164 current snap is 0 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: seq_num 1422 ranges {516096=749568,1265664=401408} 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: writing mira0435601-164 from 516096 to 1265664 tid 1 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: writing mira0435601-164 from 1265664 to 1667072 tid 2 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: read oid 438 snap -1 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: finishing write tid 2 to mira0435601-486 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: finishing write tid 3 to mira0435601-486 2014-07-01 16:49:30,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: finishing write tid 5 to mira0435601-486 2014-07-01 16:49:30,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2185: read oid 308 snap -1 2014-07-01 16:49:30,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 16:49:30,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: seq_num 1423 ranges {5316608=622592,5939200=401408} 2014-07-01 16:49:30,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: writing mira0435601-435 from 5316608 to 5939200 tid 1 2014-07-01 16:49:30,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: writing mira0435601-435 from 5939200 to 6340608 tid 2 2014-07-01 16:49:30,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2187: delete oid 225 current snap is 0 2014-07-01 16:49:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: finishing write tid 1 to mira0435601-435 2014-07-01 16:49:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: finishing write tid 2 to mira0435601-435 2014-07-01 16:49:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: finishing write tid 4 to mira0435601-435 2014-07-01 16:49:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: finishing write tid 5 to mira0435601-435 2014-07-01 16:49:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 311 (ObjNum 1423 snap 0 seq_num 1423) dirty exists 2014-07-01 16:49:30,425.425 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:30,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: finishing write tid 6 to mira0435601-486 2014-07-01 16:49:30,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 191 (ObjNum 1419 snap 0 seq_num 1419) dirty exists 2014-07-01 16:49:30,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 1 to mira0435601-164 2014-07-01 16:49:30,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2185: expect (ObjNum 1314 snap 0 seq_num 1314) 2014-07-01 16:49:31,167.167 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:48:44.721239', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882876592, u'health': u'HEALTH_OK', u'kb_used': 13904460, u'store_stats': {u'bytes_total': 7386634, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4240890}}, {u'last_updated': u'2014-07-01 07:48:54.538491', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886606904, u'health': u'HEALTH_OK', u'kb_used': 10174148, u'store_stats': {u'bytes_total': 9482575, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239679}}, {u'last_updated': u'2014-07-01 07:49:11.321414', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886593676, u'health': u'HEALTH_OK', u'kb_used': 10187376, u'store_stats': {u'bytes_total': 11579734, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 885/11394 objects degraded (7.767%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 885, u'num_pgs': 388, u'data_bytes': 21368280676, u'degraded_total': 11394, u'bytes_used': 42285559808, u'op_per_sec': 51, u'write_bytes_sec': 2320674, u'version': 1586, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 358, u'state_name': u'active+clean'}, {u'count': 11, u'state_name': u'active+recovering'}], u'read_bytes_sec': 329852, u'degraded_ratio': u'7.767', u'bytes_avail': 4956296978432}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:49:31,168.168 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:31,949.949 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:49:31,949.949 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:32,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2182: expect (ObjNum 1153 snap 0 seq_num 1153) 2014-07-01 16:49:32,755.755 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:35,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 2 to mira0435601-164 2014-07-01 16:49:35,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 4 to mira0435601-164 2014-07-01 16:49:35,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 5 to mira0435601-164 2014-07-01 16:49:35,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 237 (ObjNum 1422 snap 0 seq_num 1422) dirty exists 2014-07-01 16:49:35,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: expect (ObjNum 956 snap 0 seq_num 956) 2014-07-01 16:49:37,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: expect (ObjNum 766 snap 0 seq_num 766) 2014-07-01 16:49:38,297.297 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:38,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: done (7 left) 2014-07-01 16:49:38,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: done (6 left) 2014-07-01 16:49:38,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:2182: done (5 left) 2014-07-01 16:49:38,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: done (4 left) 2014-07-01 16:49:38,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: done (3 left) 2014-07-01 16:49:38,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2185: done (2 left) 2014-07-01 16:49:38,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: done (1 left) 2014-07-01 16:49:38,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2187: done (0 left) 2014-07-01 16:49:38,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2188: read oid 335 snap -1 2014-07-01 16:49:39,115.115 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:39,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2189: read oid 329 snap -1 2014-07-01 16:49:39,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 168 current snap is 0 2014-07-01 16:49:39,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: seq_num 1425 ranges {4071424=569344} 2014-07-01 16:49:39,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: writing mira0435601-168 from 4071424 to 4640768 tid 1 2014-07-01 16:49:39,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: delete oid 479 current snap is 0 2014-07-01 16:49:39,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2189: expect (ObjNum 1057 snap 0 seq_num 1057) 2014-07-01 16:49:40,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:2188: expect (ObjNum 716 snap 0 seq_num 716) 2014-07-01 16:49:41,102.102 INFO:teuthology.orchestra.run.mira043.stderr:testContainerSerializedInfo (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:41,737.737 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:42,410.410 INFO:teuthology.orchestra.run.mira043.stderr:testContainersOrderedByName (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:42,430.430 INFO:teuthology.orchestra.run.mira043.stderr:testInvalidAuthToken (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:42,442.442 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:48:44.721239', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882876592, u'health': u'HEALTH_OK', u'kb_used': 13904460, u'store_stats': {u'bytes_total': 7386634, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4240890}}, {u'last_updated': u'2014-07-01 07:48:54.538491', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886606904, u'health': u'HEALTH_OK', u'kb_used': 10174148, u'store_stats': {u'bytes_total': 9482575, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239679}}, {u'last_updated': u'2014-07-01 07:49:11.321414', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886593676, u'health': u'HEALTH_OK', u'kb_used': 10187376, u'store_stats': {u'bytes_total': 11579734, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'30 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 789/11240 objects degraded (7.020%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 39597866, u'degraded_objects': 789, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 20936900736, u'degraded_total': 11240, u'bytes_used': 41640058880, u'recovering_objects_per_sec': 9, u'write_bytes_sec': 415191, u'op_per_sec': 159, u'version': 1595, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 358, u'state_name': u'active+clean'}, {u'count': 11, u'state_name': u'active+recovering'}], u'read_bytes_sec': 3918369, u'degraded_ratio': u'7.020', u'bytes_avail': 4956942479360}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:49:42,443.443 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:42,585.585 INFO:teuthology.orchestra.run.mira043.stderr:testInvalidUTF8Path (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:42,585.585 INFO:teuthology.orchestra.run.mira043.stderr:testLastContainerMarker (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:42,586.586 INFO:teuthology.orchestra.run.mira043.stderr:testListingLimit (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:43,221.221 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:49:43,221.221 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:43,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 1 to mira0435601-168 2014-07-01 16:49:43,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 3 to mira0435601-168 2014-07-01 16:49:43,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 4 to mira0435601-168 2014-07-01 16:49:43,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 212 (ObjNum 1425 snap 0 seq_num 1425) dirty exists 2014-07-01 16:49:43,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2188: done (3 left) 2014-07-01 16:49:43,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:2189: done (2 left) 2014-07-01 16:49:43,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: done (1 left) 2014-07-01 16:49:43,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: done (0 left) 2014-07-01 16:49:43,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2192: delete oid 29 current snap is 0 2014-07-01 16:49:43,956.956 INFO:teuthology.orchestra.run.mira043.stderr:testMarkerLimitContainerList (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:43,979.979 INFO:teuthology.orchestra.run.mira043.stderr:testPUT (test.functional.tests.TestAccount) ... ok 2014-07-01 16:49:44,052.052 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:46,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2192: done (0 left) 2014-07-01 16:49:46,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2193: read oid 37 snap -1 2014-07-01 16:49:46,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2194: delete oid 104 current snap is 0 2014-07-01 16:49:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2194: done (1 left) 2014-07-01 16:49:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2195: read oid 459 snap -1 2014-07-01 16:49:46,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 16:49:46,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: seq_num 1429 ranges {2306048=368640} 2014-07-01 16:49:46,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: writing mira0435601-176 from 2306048 to 2674688 tid 1 2014-07-01 16:49:46,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: read oid 165 snap -1 2014-07-01 16:49:47,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: read oid 166 snap -1 2014-07-01 16:49:47,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2199: read oid 304 snap -1 2014-07-01 16:49:47,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: read oid 140 snap -1 2014-07-01 16:49:47,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 350 current snap is 0 2014-07-01 16:49:47,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: seq_num 1430 ranges {1208320=688128,1896448=401408} 2014-07-01 16:49:47,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: writing mira0435601-350 from 1208320 to 1896448 tid 1 2014-07-01 16:49:47,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: writing mira0435601-350 from 1896448 to 2297856 tid 2 2014-07-01 16:49:47,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2202: read oid 47 snap -1 2014-07-01 16:49:47,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2203: read oid 127 snap -1 2014-07-01 16:49:47,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:2193: expect (ObjNum 36 snap 0 seq_num 36) 2014-07-01 16:49:47,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: finishing write tid 1 to mira0435601-176 2014-07-01 16:49:47,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: finishing write tid 1 to mira0435601-350 2014-07-01 16:49:47,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: expect (ObjNum 1358 snap 0 seq_num 1358) 2014-07-01 16:49:49,678.678 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:50,276.276 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:51,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2199: expect (ObjNum 1300 snap 0 seq_num 1300) 2014-07-01 16:49:51,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:2202: expect (ObjNum 46 snap 0 seq_num 46) 2014-07-01 16:49:51,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: finishing write tid 2 to mira0435601-350 2014-07-01 16:49:51,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: finishing write tid 4 to mira0435601-350 2014-07-01 16:49:51,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: finishing write tid 5 to mira0435601-350 2014-07-01 16:49:51,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 251 (ObjNum 1430 snap 0 seq_num 1430) dirty exists 2014-07-01 16:49:51,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: expect (ObjNum 864 snap 0 seq_num 864) 2014-07-01 16:49:53,056.056 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:53,757.757 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:49:44.721491', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882803524, u'health': u'HEALTH_OK', u'kb_used': 13977528, u'store_stats': {u'bytes_total': 6564525, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 6498973}}, {u'last_updated': u'2014-07-01 07:48:54.538491', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886606904, u'health': u'HEALTH_OK', u'kb_used': 10174148, u'store_stats': {u'bytes_total': 9482575, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239679}}, {u'last_updated': u'2014-07-01 07:49:11.321414', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886593676, u'health': u'HEALTH_OK', u'kb_used': 10187376, u'store_stats': {u'bytes_total': 11579734, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'29 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 759/10900 objects degraded (6.963%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 10416417, u'degraded_objects': 759, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 20221790845, u'degraded_total': 10900, u'bytes_used': 40233684992, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 544725, u'op_per_sec': 47, u'version': 1602, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 359, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2453554, u'degraded_ratio': u'6.963', u'bytes_avail': 4958348853248}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:49:53,758.758 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:49:54,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: finishing write tid 3 to mira0435601-176 2014-07-01 16:49:54,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2195: expect (ObjNum 1364 snap 0 seq_num 1364) 2014-07-01 16:49:54,545.545 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:49:54,545.545 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:49:55,346.346 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:49:58,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: expect (ObjNum 1076 snap 0 seq_num 1076) 2014-07-01 16:50:00,925.925 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:01,713.713 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:02,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: finishing write tid 4 to mira0435601-176 2014-07-01 16:50:02,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 205 (ObjNum 1429 snap 0 seq_num 1429) dirty exists 2014-07-01 16:50:02,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2203: expect (ObjNum 661 snap 0 seq_num 661) 2014-07-01 16:50:04,265.265 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:04,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2193: done (9 left) 2014-07-01 16:50:04,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2195: done (8 left) 2014-07-01 16:50:04,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: done (7 left) 2014-07-01 16:50:04,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: done (6 left) 2014-07-01 16:50:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: done (5 left) 2014-07-01 16:50:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2199: done (4 left) 2014-07-01 16:50:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: done (3 left) 2014-07-01 16:50:04,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: done (2 left) 2014-07-01 16:50:04,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2202: done (1 left) 2014-07-01 16:50:04,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2203: done (0 left) 2014-07-01 16:50:04,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 37 current snap is 0 2014-07-01 16:50:04,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: seq_num 1431 ranges {765952=561152,1327104=401408} 2014-07-01 16:50:04,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: writing mira0435601-37 from 765952 to 1327104 tid 1 2014-07-01 16:50:04,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: writing mira0435601-37 from 1327104 to 1728512 tid 2 2014-07-01 16:50:04,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 329 current snap is 0 2014-07-01 16:50:04,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: seq_num 1432 ranges {2195456=585728,2781184=589824,3371008=581632,3952640=434176} 2014-07-01 16:50:04,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: writing mira0435601-329 from 2195456 to 2781184 tid 1 2014-07-01 16:50:04,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: writing mira0435601-329 from 2781184 to 3371008 tid 2 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: writing mira0435601-329 from 3371008 to 3952640 tid 3 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: writing mira0435601-329 from 3952640 to 4386816 tid 4 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 98 current snap is 0 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: seq_num 1433 ranges {2949120=458752,3407872=417792,3825664=651264,4476928=536576} 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: writing mira0435601-98 from 2949120 to 3407872 tid 1 2014-07-01 16:50:04,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: writing mira0435601-98 from 3407872 to 3825664 tid 2 2014-07-01 16:50:04,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: writing mira0435601-98 from 3825664 to 4476928 tid 3 2014-07-01 16:50:04,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: writing mira0435601-98 from 4476928 to 5013504 tid 4 2014-07-01 16:50:04,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: read oid 190 snap -1 2014-07-01 16:50:04,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 306 current snap is 0 2014-07-01 16:50:05,010.010 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:49:44.721491', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882803524, u'health': u'HEALTH_OK', u'kb_used': 13977528, u'store_stats': {u'bytes_total': 6564525, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 6498973}}, {u'last_updated': u'2014-07-01 07:49:54.538899', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886594120, u'health': u'HEALTH_OK', u'kb_used': 10186932, u'store_stats': {u'bytes_total': 4609740, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:49:11.321414', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886593676, u'health': u'HEALTH_OK', u'kb_used': 10187376, u'store_stats': {u'bytes_total': 11579734, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4239686}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'9 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 603/10424 objects degraded (5.785%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 603, u'num_pgs': 388, u'data_bytes': 19403257088, u'degraded_total': 10424, u'bytes_used': 38982135808, u'op_per_sec': 53, u'write_bytes_sec': 2040, u'version': 1612, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 9, u'state_name': u'active+recovering'}], u'read_bytes_sec': 19041, u'degraded_ratio': u'5.785', u'bytes_avail': 4959600402432}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:50:05,011.011 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:05,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: seq_num 1434 ranges {2318336=380928} 2014-07-01 16:50:05,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: writing mira0435601-306 from 2318336 to 2699264 tid 1 2014-07-01 16:50:05,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: read oid 475 snap -1 2014-07-01 16:50:05,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: read oid 20 snap -1 2014-07-01 16:50:05,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: read oid 251 snap -1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2212: read oid 166 snap -1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: read oid 456 snap -1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: read oid 441 snap -1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2215: read oid 332 snap -1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 29 current snap is 0 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: seq_num 1435 ranges {0=405504,405504=585728,991232=618496} 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: writing mira0435601-29 from 0 to 405504 tid 1 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: writing mira0435601-29 from 405504 to 991232 tid 2 2014-07-01 16:50:05,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: writing mira0435601-29 from 991232 to 1609728 tid 3 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2217: delete oid 445 current snap is 0 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: finishing write tid 1 to mira0435601-306 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: finishing write tid 3 to mira0435601-306 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: finishing write tid 4 to mira0435601-306 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 253 (ObjNum 1434 snap 0 seq_num 1434) dirty exists 2014-07-01 16:50:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: expect (ObjNum 1156 snap 0 seq_num 1156) 2014-07-01 16:50:05,796.796 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:50:05,797.797 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:05,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: expect (ObjNum 540 snap 0 seq_num 540) 2014-07-01 16:50:06,609.609 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:08,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: finishing write tid 1 to mira0435601-37 2014-07-01 16:50:08,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 1 to mira0435601-329 2014-07-01 16:50:08,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: finishing write tid 2 to mira0435601-37 2014-07-01 16:50:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 2 to mira0435601-329 2014-07-01 16:50:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: finishing write tid 4 to mira0435601-37 2014-07-01 16:50:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 3 to mira0435601-329 2014-07-01 16:50:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 4 to mira0435601-329 2014-07-01 16:50:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 6 to mira0435601-329 2014-07-01 16:50:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 1 to mira0435601-98 2014-07-01 16:50:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 2 to mira0435601-98 2014-07-01 16:50:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: expect (ObjNum 1334 snap 0 seq_num 1334) 2014-07-01 16:50:11,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 3 to mira0435601-98 2014-07-01 16:50:11,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 4 to mira0435601-98 2014-07-01 16:50:11,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 6 to mira0435601-98 2014-07-01 16:50:11,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: expect (ObjNum 507 snap 0 seq_num 507) 2014-07-01 16:50:12,152.152 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:12,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: expect (ObjNum 1222 snap 0 seq_num 1222) 2014-07-01 16:50:12,920.920 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:14,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:2212: expect (ObjNum 864 snap 0 seq_num 864) 2014-07-01 16:50:15,531.531 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:16,247.247 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 4, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.022436', u'skew': u'0.000184', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.010708', u'skew': u'0.000153', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:49:44.721491', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882803524, u'health': u'HEALTH_OK', u'kb_used': 13977528, u'store_stats': {u'bytes_total': 6564525, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 6498973}}, {u'last_updated': u'2014-07-01 07:49:54.538899', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886594120, u'health': u'HEALTH_OK', u'kb_used': 10186932, u'store_stats': {u'bytes_total': 4609740, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:50:11.321898', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886585328, u'health': u'HEALTH_OK', u'kb_used': 10195724, u'store_stats': {u'bytes_total': 5658376, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'18 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 563/10110 objects degraded (5.569%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 563, u'num_pgs': 388, u'data_bytes': 19001898930, u'degraded_total': 10110, u'bytes_used': 38205546496, u'op_per_sec': 285, u'write_bytes_sec': 10697, u'version': 1621, u'pgs_by_state': [{u'count': 18, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 135499, u'degraded_ratio': u'5.569', u'bytes_avail': 4960376991744}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:50:16,248.248 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:16,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: expect (ObjNum 1280 snap 0 seq_num 1280) 2014-07-01 16:50:17,021.021 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:50:17,021.021 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:17,847.847 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:18,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: finishing write tid 5 to mira0435601-37 2014-07-01 16:50:18,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 109 (ObjNum 1431 snap 0 seq_num 1431) dirty exists 2014-07-01 16:50:18,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: finishing write tid 7 to mira0435601-329 2014-07-01 16:50:18,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 329 v 277 (ObjNum 1432 snap 0 seq_num 1432) dirty exists 2014-07-01 16:50:18,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: finishing write tid 7 to mira0435601-98 2014-07-01 16:50:18,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 259 (ObjNum 1433 snap 0 seq_num 1433) dirty exists 2014-07-01 16:50:18,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2215: expect (ObjNum 1301 snap 0 seq_num 1301) 2014-07-01 16:50:20,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: finishing write tid 1 to mira0435601-29 2014-07-01 16:50:20,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: finishing write tid 2 to mira0435601-29 2014-07-01 16:50:20,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: finishing write tid 3 to mira0435601-29 2014-07-01 16:50:20,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: finishing write tid 5 to mira0435601-29 2014-07-01 16:50:20,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: finishing write tid 6 to mira0435601-29 2014-07-01 16:50:20,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 196 (ObjNum 1435 snap 0 seq_num 1435) dirty exists 2014-07-01 16:50:20,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: done (13 left) 2014-07-01 16:50:20,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: done (12 left) 2014-07-01 16:50:20,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: done (11 left) 2014-07-01 16:50:20,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: done (10 left) 2014-07-01 16:50:20,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: done (9 left) 2014-07-01 16:50:20,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: done (8 left) 2014-07-01 16:50:20,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: done (7 left) 2014-07-01 16:50:20,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: done (6 left) 2014-07-01 16:50:20,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2212: done (5 left) 2014-07-01 16:50:20,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: done (4 left) 2014-07-01 16:50:20,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: done (3 left) 2014-07-01 16:50:20,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2215: done (2 left) 2014-07-01 16:50:20,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: done (1 left) 2014-07-01 16:50:20,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2217: done (0 left) 2014-07-01 16:50:20,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2218: read oid 8 snap -1 2014-07-01 16:50:20,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2219: read oid 401 snap -1 2014-07-01 16:50:20,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:50:20,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: seq_num 1437 ranges {9879552=552960,10432512=802816} 2014-07-01 16:50:20,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: writing mira0435601-439 from 9879552 to 10432512 tid 1 2014-07-01 16:50:20,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: writing mira0435601-439 from 10432512 to 11235328 tid 2 2014-07-01 16:50:20,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 67 current snap is 0 2014-07-01 16:50:20,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: seq_num 1438 ranges {3334144=176128} 2014-07-01 16:50:20,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: writing mira0435601-67 from 3334144 to 3510272 tid 1 2014-07-01 16:50:20,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 16:50:20,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: seq_num 1439 ranges {2682880=745472,3428352=565248,3993600=598016,4591616=40960} 2014-07-01 16:50:20,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: writing mira0435601-296 from 2682880 to 3428352 tid 1 2014-07-01 16:50:20,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: writing mira0435601-296 from 3428352 to 3993600 tid 2 2014-07-01 16:50:20,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: writing mira0435601-296 from 3993600 to 4591616 tid 3 2014-07-01 16:50:20,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: writing mira0435601-296 from 4591616 to 4632576 tid 4 2014-07-01 16:50:20,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: read oid 110 snap -1 2014-07-01 16:50:20,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: finishing write tid 1 to mira0435601-439 2014-07-01 16:50:20,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: delete oid 7 current snap is 0 2014-07-01 16:50:20,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: finishing write tid 2 to mira0435601-439 2014-07-01 16:50:20,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: finishing write tid 4 to mira0435601-439 2014-07-01 16:50:20,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: finishing write tid 5 to mira0435601-439 2014-07-01 16:50:20,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 239 (ObjNum 1437 snap 0 seq_num 1437) dirty exists 2014-07-01 16:50:21,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2219: expect (ObjNum 1346 snap 0 seq_num 1346) 2014-07-01 16:50:21,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:2218: expect (ObjNum 1110 snap 0 seq_num 1110) 2014-07-01 16:50:22,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 1 to mira0435601-296 2014-07-01 16:50:22,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: finishing write tid 1 to mira0435601-67 2014-07-01 16:50:22,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 2 to mira0435601-296 2014-07-01 16:50:22,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: finishing write tid 3 to mira0435601-67 2014-07-01 16:50:22,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 3 to mira0435601-296 2014-07-01 16:50:22,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: finishing write tid 4 to mira0435601-67 2014-07-01 16:50:22,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 201 (ObjNum 1438 snap 0 seq_num 1438) dirty exists 2014-07-01 16:50:22,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:2218: done (6 left) 2014-07-01 16:50:22,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2219: done (5 left) 2014-07-01 16:50:22,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: done (4 left) 2014-07-01 16:50:22,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: done (3 left) 2014-07-01 16:50:22,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: done (2 left) 2014-07-01 16:50:22,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2225: read oid 50 snap -1 2014-07-01 16:50:22,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 4 to mira0435601-296 2014-07-01 16:50:22,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: expect (ObjNum 973 snap 0 seq_num 973) 2014-07-01 16:50:22,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 6 to mira0435601-296 2014-07-01 16:50:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: finishing write tid 7 to mira0435601-296 2014-07-01 16:50:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 191 (ObjNum 1439 snap 0 seq_num 1439) dirty exists 2014-07-01 16:50:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2225: expect (ObjNum 1390 snap 0 seq_num 1390) 2014-07-01 16:50:23,231.231 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:24,006.006 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:26,598.598 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:27,304.304 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:49:44.721491', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882803524, u'health': u'HEALTH_OK', u'kb_used': 13977528, u'store_stats': {u'bytes_total': 6564525, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 6498973}}, {u'last_updated': u'2014-07-01 07:49:54.538899', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886594120, u'health': u'HEALTH_OK', u'kb_used': 10186932, u'store_stats': {u'bytes_total': 4609740, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:50:11.321898', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886585328, u'health': u'HEALTH_OK', u'kb_used': 10195724, u'store_stats': {u'bytes_total': 5658376, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'18 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 466/9827 objects degraded (4.742%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 23491012, u'degraded_objects': 466, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 18517042242, u'degraded_total': 9827, u'bytes_used': 37334929408, u'recovering_objects_per_sec': 5, u'write_bytes_sec': 662184, u'op_per_sec': 29, u'version': 1631, u'pgs_by_state': [{u'count': 18, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 413810, u'degraded_ratio': u'4.742', u'bytes_avail': 4961247608832}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:50:27,305.305 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:27,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: done (2 left) 2014-07-01 16:50:27,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: done (1 left) 2014-07-01 16:50:27,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:2225: done (0 left) 2014-07-01 16:50:27,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 214 current snap is 0 2014-07-01 16:50:27,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: seq_num 1441 ranges {3796992=798720,4595712=782336,5378048=421888} 2014-07-01 16:50:27,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: writing mira0435601-214 from 3796992 to 4595712 tid 1 2014-07-01 16:50:27,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: writing mira0435601-214 from 4595712 to 5378048 tid 2 2014-07-01 16:50:27,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: writing mira0435601-214 from 5378048 to 5799936 tid 3 2014-07-01 16:50:27,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: read oid 428 snap -1 2014-07-01 16:50:27,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 157 current snap is 0 2014-07-01 16:50:27,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: seq_num 1442 ranges {1806336=671744,2478080=716800,3194880=421888,3616768=65536} 2014-07-01 16:50:27,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0435601-157 from 1806336 to 2478080 tid 1 2014-07-01 16:50:27,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0435601-157 from 2478080 to 3194880 tid 2 2014-07-01 16:50:27,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0435601-157 from 3194880 to 3616768 tid 3 2014-07-01 16:50:27,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0435601-157 from 3616768 to 3682304 tid 4 2014-07-01 16:50:27,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2229: delete oid 238 current snap is 0 2014-07-01 16:50:27,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: expect (ObjNum 1327 snap 0 seq_num 1327) 2014-07-01 16:50:28,099.099 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:50:28,100.100 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:28,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: finishing write tid 1 to mira0435601-214 2014-07-01 16:50:28,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: finishing write tid 2 to mira0435601-214 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: finishing write tid 3 to mira0435601-214 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: finishing write tid 5 to mira0435601-214 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: done (3 left) 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2229: done (2 left) 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2230: read oid 363 snap -1 2014-07-01 16:50:28,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 181 current snap is 0 2014-07-01 16:50:28,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: seq_num 1444 ranges {2138112=618496,2756608=778240,3534848=24576} 2014-07-01 16:50:28,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: writing mira0435601-181 from 2138112 to 2756608 tid 1 2014-07-01 16:50:28,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: writing mira0435601-181 from 2756608 to 3534848 tid 2 2014-07-01 16:50:28,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: writing mira0435601-181 from 3534848 to 3559424 tid 3 2014-07-01 16:50:28,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 378 current snap is 0 2014-07-01 16:50:28,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: seq_num 1445 ranges {823296=491520,1314816=606208,1921024=196608} 2014-07-01 16:50:28,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0435601-378 from 823296 to 1314816 tid 1 2014-07-01 16:50:28,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0435601-378 from 1314816 to 1921024 tid 2 2014-07-01 16:50:28,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0435601-378 from 1921024 to 2117632 tid 3 2014-07-01 16:50:28,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: read oid 100 snap -1 2014-07-01 16:50:28,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: finishing write tid 1 to mira0435601-181 2014-07-01 16:50:28,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: finishing write tid 2 to mira0435601-181 2014-07-01 16:50:28,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: finishing write tid 3 to mira0435601-181 2014-07-01 16:50:28,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: finishing write tid 5 to mira0435601-181 2014-07-01 16:50:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: finishing write tid 6 to mira0435601-181 2014-07-01 16:50:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 315 (ObjNum 1444 snap 0 seq_num 1444) dirty exists 2014-07-01 16:50:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: finishing write tid 6 to mira0435601-214 2014-07-01 16:50:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 122 (ObjNum 1441 snap 0 seq_num 1441) dirty exists 2014-07-01 16:50:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: done (5 left) 2014-07-01 16:50:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2230: done (4 left) 2014-07-01 16:50:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: done (3 left) 2014-07-01 16:50:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2234: read oid 490 snap -1 2014-07-01 16:50:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2235: delete oid 419 current snap is 0 2014-07-01 16:50:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2235: done (4 left) 2014-07-01 16:50:28,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 345 current snap is 0 2014-07-01 16:50:28,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: seq_num 1447 ranges {2043904=790528,2834432=774144,3608576=696320,4304896=135168} 2014-07-01 16:50:28,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: writing mira0435601-345 from 2043904 to 2834432 tid 1 2014-07-01 16:50:28,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: writing mira0435601-345 from 2834432 to 3608576 tid 2 2014-07-01 16:50:28,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: writing mira0435601-345 from 3608576 to 4304896 tid 3 2014-07-01 16:50:28,850.850 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:28,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: writing mira0435601-345 from 4304896 to 4440064 tid 4 2014-07-01 16:50:28,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2237: read oid 321 snap -1 2014-07-01 16:50:28,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2238: read oid 385 snap -1 2014-07-01 16:50:28,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2234: expect (ObjNum 879 snap 0 seq_num 879) 2014-07-01 16:50:31,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: expect (ObjNum 1198 snap 0 seq_num 1198) 2014-07-01 16:50:34,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2237: expect (ObjNum 994 snap 0 seq_num 994) 2014-07-01 16:50:34,230.230 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:34,990.990 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:36,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 1 to mira0435601-157 2014-07-01 16:50:36,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 2 to mira0435601-157 2014-07-01 16:50:36,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 1 to mira0435601-378 2014-07-01 16:50:36,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 3 to mira0435601-157 2014-07-01 16:50:36,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 2 to mira0435601-378 2014-07-01 16:50:36,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 3 to mira0435601-378 2014-07-01 16:50:36,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 5 to mira0435601-378 2014-07-01 16:50:36,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 4 to mira0435601-157 2014-07-01 16:50:36,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 1 to mira0435601-345 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 2 to mira0435601-345 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 6 to mira0435601-157 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 3 to mira0435601-345 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 6 to mira0435601-378 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 378 v 136 (ObjNum 1445 snap 0 seq_num 1445) dirty exists 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 7 to mira0435601-157 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 245 (ObjNum 1442 snap 0 seq_num 1442) dirty exists 2014-07-01 16:50:36,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2238: expect (ObjNum 384 snap 0 seq_num 384) 2014-07-01 16:50:37,584.584 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 4 to mira0435601-345 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 6 to mira0435601-345 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: finishing write tid 7 to mira0435601-345 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 345 v 205 (ObjNum 1447 snap 0 seq_num 1447) dirty exists 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: done (6 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: done (5 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: done (4 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2234: done (3 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: done (2 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2237: done (1 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2238: done (0 left) 2014-07-01 16:50:37,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: read oid 142 snap -1 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2240: read oid 426 snap -1 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: read oid 315 snap -1 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 344 current snap is 0 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: seq_num 1448 ranges {815104=712704} 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: writing mira0435601-344 from 815104 to 1527808 tid 1 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2243: read oid 58 snap -1 2014-07-01 16:50:37,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2240: expect (ObjNum 425 snap 0 seq_num 425) 2014-07-01 16:50:38,297.297 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:49:44.721491', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882803524, u'health': u'HEALTH_OK', u'kb_used': 13977528, u'store_stats': {u'bytes_total': 6564525, u'bytes_log': 0, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 6498973}}, {u'last_updated': u'2014-07-01 07:49:54.538899', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886594120, u'health': u'HEALTH_OK', u'kb_used': 10186932, u'store_stats': {u'bytes_total': 4609740, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:50:11.321898', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886585328, u'health': u'HEALTH_OK', u'kb_used': 10195724, u'store_stats': {u'bytes_total': 5658376, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'18 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 416/9541 objects degraded (4.360%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 10415476, u'degraded_objects': 416, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 17973879017, u'degraded_total': 9541, u'bytes_used': 36587315200, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 2034, u'op_per_sec': 16, u'version': 1639, u'pgs_by_state': [{u'count': 18, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1017, u'degraded_ratio': u'4.360', u'bytes_avail': 4961995223040}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:50:38,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:38,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: finishing write tid 1 to mira0435601-344 2014-07-01 16:50:38,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: finishing write tid 3 to mira0435601-344 2014-07-01 16:50:38,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: finishing write tid 4 to mira0435601-344 2014-07-01 16:50:38,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 317 (ObjNum 1448 snap 0 seq_num 1448) dirty exists 2014-07-01 16:50:38,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2243: expect (ObjNum 1147 snap 0 seq_num 1147) 2014-07-01 16:50:39,040.040 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:50:39,040.040 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:39,863.863 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:42,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 16:50:42,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: done (4 left) 2014-07-01 16:50:42,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2240: done (3 left) 2014-07-01 16:50:42,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: done (2 left) 2014-07-01 16:50:42,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: done (1 left) 2014-07-01 16:50:42,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:2243: done (0 left) 2014-07-01 16:50:42,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 387 current snap is 0 2014-07-01 16:50:42,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: seq_num 1449 ranges {634880=585728} 2014-07-01 16:50:42,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: writing mira0435601-387 from 634880 to 1220608 tid 1 2014-07-01 16:50:42,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: read oid 351 snap -1 2014-07-01 16:50:42,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: delete oid 122 current snap is 0 2014-07-01 16:50:42,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: expect (ObjNum 1223 snap 0 seq_num 1223) 2014-07-01 16:50:45,388.388 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:45,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: finishing write tid 1 to mira0435601-387 2014-07-01 16:50:45,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: finishing write tid 3 to mira0435601-387 2014-07-01 16:50:45,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: finishing write tid 4 to mira0435601-387 2014-07-01 16:50:45,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 130 (ObjNum 1449 snap 0 seq_num 1449) dirty exists 2014-07-01 16:50:45,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: done (2 left) 2014-07-01 16:50:45,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: done (1 left) 2014-07-01 16:50:45,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: done (0 left) 2014-07-01 16:50:45,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: read oid 172 snap -1 2014-07-01 16:50:45,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 325 current snap is 0 2014-07-01 16:50:45,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: seq_num 1451 ranges {4530176=483328,5013504=753664,5767168=450560} 2014-07-01 16:50:45,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: writing mira0435601-325 from 4530176 to 5013504 tid 1 2014-07-01 16:50:45,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: writing mira0435601-325 from 5013504 to 5767168 tid 2 2014-07-01 16:50:45,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: writing mira0435601-325 from 5767168 to 6217728 tid 3 2014-07-01 16:50:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 16:50:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: seq_num 1452 ranges {1667072=802816,2469888=741376,3211264=659456,3870720=204800} 2014-07-01 16:50:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: writing mira0435601-492 from 1667072 to 2469888 tid 1 2014-07-01 16:50:45,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: writing mira0435601-492 from 2469888 to 3211264 tid 2 2014-07-01 16:50:45,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: writing mira0435601-492 from 3211264 to 3870720 tid 3 2014-07-01 16:50:45,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: writing mira0435601-492 from 3870720 to 4075520 tid 4 2014-07-01 16:50:45,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 46 current snap is 0 2014-07-01 16:50:45,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: seq_num 1453 ranges {2813952=430080,3244032=569344,3813376=233472} 2014-07-01 16:50:45,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: writing mira0435601-46 from 2813952 to 3244032 tid 1 2014-07-01 16:50:45,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: writing mira0435601-46 from 3244032 to 3813376 tid 2 2014-07-01 16:50:45,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: writing mira0435601-46 from 3813376 to 4046848 tid 3 2014-07-01 16:50:45,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: read oid 277 snap -1 2014-07-01 16:50:45,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 16:50:45,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: seq_num 1454 ranges {4538368=704512,5242880=401408} 2014-07-01 16:50:45,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: writing mira0435601-394 from 4538368 to 5242880 tid 1 2014-07-01 16:50:45,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: writing mira0435601-394 from 5242880 to 5644288 tid 2 2014-07-01 16:50:45,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2253: read oid 8 snap -1 2014-07-01 16:50:45,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 222 current snap is 0 2014-07-01 16:50:45,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: seq_num 1455 ranges {3100672=729088,3829760=581632,4411392=434176,4845568=512000,5357568=77824} 2014-07-01 16:50:45,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: writing mira0435601-222 from 3100672 to 3829760 tid 1 2014-07-01 16:50:45,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: writing mira0435601-222 from 3829760 to 4411392 tid 2 2014-07-01 16:50:45,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: writing mira0435601-222 from 4411392 to 4845568 tid 3 2014-07-01 16:50:45,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: writing mira0435601-222 from 4845568 to 5357568 tid 4 2014-07-01 16:50:45,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: writing mira0435601-222 from 5357568 to 5435392 tid 5 2014-07-01 16:50:45,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2255: read oid 185 snap -1 2014-07-01 16:50:45,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2256: read oid 126 snap -1 2014-07-01 16:50:45,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 210 current snap is 0 2014-07-01 16:50:45,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: seq_num 1456 ranges {802816=602112,1404928=565248,1970176=659456,2629632=380928} 2014-07-01 16:50:45,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0435601-210 from 802816 to 1404928 tid 1 2014-07-01 16:50:45,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0435601-210 from 1404928 to 1970176 tid 2 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0435601-210 from 1970176 to 2629632 tid 3 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0435601-210 from 2629632 to 3010560 tid 4 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2258: read oid 249 snap -1 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 1 to mira0435601-210 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 2 to mira0435601-210 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2256: done (11 left) 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2259: read oid 417 snap -1 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 263 current snap is 0 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: seq_num 1457 ranges {1503232=524288} 2014-07-01 16:50:45,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: writing mira0435601-263 from 1503232 to 2027520 tid 1 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: read oid 129 snap -1 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 3 to mira0435601-210 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 4 to mira0435601-210 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 6 to mira0435601-210 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 7 to mira0435601-210 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 210 v 326 (ObjNum 1456 snap 0 seq_num 1456) dirty exists 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: done (13 left) 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2258: done (12 left) 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2259: done (11 left) 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: read oid 254 snap -1 2014-07-01 16:50:45,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 278 current snap is 0 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: seq_num 1458 ranges {3768320=548864,4317184=733184,5050368=471040} 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: writing mira0435601-278 from 3768320 to 4317184 tid 1 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: writing mira0435601-278 from 4317184 to 5050368 tid 2 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: writing mira0435601-278 from 5050368 to 5521408 tid 3 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: read oid 97 snap -1 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: finishing write tid 1 to mira0435601-263 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: finishing write tid 3 to mira0435601-263 2014-07-01 16:50:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: finishing write tid 4 to mira0435601-263 2014-07-01 16:50:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 241 (ObjNum 1457 snap 0 seq_num 1457) dirty exists 2014-07-01 16:50:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: done (13 left) 2014-07-01 16:50:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: read oid 292 snap -1 2014-07-01 16:50:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2266: read oid 407 snap -1 2014-07-01 16:50:45,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 148 current snap is 0 2014-07-01 16:50:45,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: seq_num 1459 ranges {634880=421888,1056768=561152,1617920=434176,2052096=208896} 2014-07-01 16:50:46,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0435601-148 from 634880 to 1056768 tid 1 2014-07-01 16:50:46,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0435601-148 from 1056768 to 1617920 tid 2 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0435601-148 from 1617920 to 2052096 tid 3 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0435601-148 from 2052096 to 2260992 tid 4 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2266: done (15 left) 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2268: delete oid 162 current snap is 0 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 1 to mira0435601-325 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 1 to mira0435601-148 2014-07-01 16:50:46,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 2 to mira0435601-148 2014-07-01 16:50:46,166.166 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:46,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: expect (ObjNum 1277 snap 0 seq_num 1277) 2014-07-01 16:50:48,120.120 INFO:teuthology.orchestra.run.mira043.stderr:testGetRequest (test.functional.tests.TestAccountNoContainers) ... ok 2014-07-01 16:50:48,140.140 INFO:teuthology.orchestra.run.mira043.stderr:testGetRequest (test.functional.tests.TestAccountNoContainersUTF8) ... ok 2014-07-01 16:50:49,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: expect (ObjNum 1087 snap 0 seq_num 1087) 2014-07-01 16:50:52,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 3 to mira0435601-148 2014-07-01 16:50:52,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2253: expect (ObjNum 1110 snap 0 seq_num 1110) 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 4 to mira0435601-148 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 6 to mira0435601-148 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 7 to mira0435601-148 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 322 (ObjNum 1459 snap 0 seq_num 1459) dirty exists 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 2 to mira0435601-325 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 1 to mira0435601-394 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 2 to mira0435601-394 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 3 to mira0435601-325 2014-07-01 16:50:53,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 5 to mira0435601-325 2014-07-01 16:50:53,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 4 to mira0435601-394 2014-07-01 16:50:53,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 6 to mira0435601-325 2014-07-01 16:50:53,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 159 (ObjNum 1451 snap 0 seq_num 1451) dirty exists 2014-07-01 16:50:53,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: expect (ObjNum 1282 snap 0 seq_num 1282) 2014-07-01 16:50:53,740.740 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: finishing write tid 1 to mira0435601-278 2014-07-01 16:50:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: finishing write tid 2 to mira0435601-278 2014-07-01 16:50:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: finishing write tid 3 to mira0435601-278 2014-07-01 16:50:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: finishing write tid 5 to mira0435601-278 2014-07-01 16:50:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 5 to mira0435601-394 2014-07-01 16:50:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 188 (ObjNum 1454 snap 0 seq_num 1454) dirty exists 2014-07-01 16:50:54,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: done (15 left) 2014-07-01 16:50:54,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: done (14 left) 2014-07-01 16:50:54,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: done (13 left) 2014-07-01 16:50:54,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2253: done (12 left) 2014-07-01 16:50:54,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: done (11 left) 2014-07-01 16:50:54,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: done (10 left) 2014-07-01 16:50:54,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: done (9 left) 2014-07-01 16:50:54,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2268: done (8 left) 2014-07-01 16:50:54,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2269: read oid 369 snap -1 2014-07-01 16:50:54,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 1 to mira0435601-46 2014-07-01 16:50:54,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 1 to mira0435601-492 2014-07-01 16:50:54,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 2 to mira0435601-46 2014-07-01 16:50:54,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: finishing write tid 6 to mira0435601-278 2014-07-01 16:50:54,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 263 (ObjNum 1458 snap 0 seq_num 1458) dirty exists 2014-07-01 16:50:54,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: expect (ObjNum 1011 snap 0 seq_num 1011) 2014-07-01 16:50:54,461.461 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:50:44.721894', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882745560, u'health': u'HEALTH_OK', u'kb_used': 14035492, u'store_stats': {u'bytes_total': 6707415, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:49:54.538899', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886594120, u'health': u'HEALTH_OK', u'kb_used': 10186932, u'store_stats': {u'bytes_total': 4609740, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:50:11.321898', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886585328, u'health': u'HEALTH_OK', u'kb_used': 10195724, u'store_stats': {u'bytes_total': 5658376, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'18 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 308/9082 objects degraded (3.391%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 308, u'num_pgs': 388, u'data_bytes': 17103955672, u'degraded_total': 9082, u'bytes_used': 34956132352, u'op_per_sec': 11, u'write_bytes_sec': 3057, u'version': 1653, u'pgs_by_state': [{u'count': 18, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2038, u'degraded_ratio': u'3.391', u'bytes_avail': 4963626405888}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:50:54,462.462 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:50:55,199.199 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:50:55,200.200 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:50:55,964.964 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:50:56,835.835 INFO:teuthology.orchestra.run.mira043.stderr:testAccountHead (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:50:56,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 2 to mira0435601-492 2014-07-01 16:50:56,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 3 to mira0435601-492 2014-07-01 16:50:56,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 4 to mira0435601-492 2014-07-01 16:50:56,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 6 to mira0435601-492 2014-07-01 16:50:56,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: finishing write tid 7 to mira0435601-492 2014-07-01 16:50:56,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 164 (ObjNum 1452 snap 0 seq_num 1452) dirty exists 2014-07-01 16:50:56,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 3 to mira0435601-46 2014-07-01 16:50:56,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 5 to mira0435601-46 2014-07-01 16:50:56,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 6 to mira0435601-46 2014-07-01 16:50:56,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 134 (ObjNum 1453 snap 0 seq_num 1453) dirty exists 2014-07-01 16:50:56,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: expect (ObjNum 909 snap 0 seq_num 909) 2014-07-01 16:50:57,261.261 INFO:teuthology.orchestra.run.mira043.stderr:testContainerListing (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:50:58,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 1 to mira0435601-222 2014-07-01 16:50:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 2 to mira0435601-222 2014-07-01 16:50:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 3 to mira0435601-222 2014-07-01 16:50:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 4 to mira0435601-222 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 5 to mira0435601-222 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 7 to mira0435601-222 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: done (8 left) 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: done (7 left) 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: done (6 left) 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: done (5 left) 2014-07-01 16:50:58,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: done (4 left) 2014-07-01 16:50:58,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 283 current snap is 0 2014-07-01 16:50:58,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: seq_num 1461 ranges {0=770048,770048=401408} 2014-07-01 16:50:58,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: writing mira0435601-283 from 0 to 770048 tid 1 2014-07-01 16:50:58,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: writing mira0435601-283 from 770048 to 1171456 tid 2 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 90 current snap is 0 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: seq_num 1462 ranges {1118208=643072,1761280=401408} 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: writing mira0435601-90 from 1118208 to 1761280 tid 1 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: writing mira0435601-90 from 1761280 to 2162688 tid 2 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2272: delete oid 56 current snap is 0 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: finishing write tid 8 to mira0435601-222 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 251 (ObjNum 1455 snap 0 seq_num 1455) dirty exists 2014-07-01 16:50:58,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:2255: expect (ObjNum 184 snap 0 seq_num 184) 2014-07-01 16:50:58,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: expect (ObjNum 929 snap 0 seq_num 929) 2014-07-01 16:51:01,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2269: expect (ObjNum 1297 snap 0 seq_num 1297) 2014-07-01 16:51:02,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: finishing write tid 1 to mira0435601-283 2014-07-01 16:51:02,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: finishing write tid 2 to mira0435601-283 2014-07-01 16:51:02,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: finishing write tid 4 to mira0435601-283 2014-07-01 16:51:02,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: finishing write tid 5 to mira0435601-283 2014-07-01 16:51:02,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 325 (ObjNum 1461 snap 0 seq_num 1461) dirty exists 2014-07-01 16:51:02,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: finishing write tid 1 to mira0435601-90 2014-07-01 16:51:02,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: finishing write tid 2 to mira0435601-90 2014-07-01 16:51:02,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: finishing write tid 4 to mira0435601-90 2014-07-01 16:51:02,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: finishing write tid 5 to mira0435601-90 2014-07-01 16:51:02,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 90 v 112 (ObjNum 1462 snap 0 seq_num 1462) dirty exists 2014-07-01 16:51:02,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: done (6 left) 2014-07-01 16:51:02,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2255: done (5 left) 2014-07-01 16:51:02,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: done (4 left) 2014-07-01 16:51:02,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:2269: done (3 left) 2014-07-01 16:51:02,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: done (2 left) 2014-07-01 16:51:02,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: done (1 left) 2014-07-01 16:51:02,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2272: done (0 left) 2014-07-01 16:51:02,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 427 current snap is 0 2014-07-01 16:51:02,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: seq_num 1464 ranges {0=540672,540672=770048,1310720=552960,1863680=282624} 2014-07-01 16:51:02,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: writing mira0435601-427 from 0 to 540672 tid 1 2014-07-01 16:51:02,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: writing mira0435601-427 from 540672 to 1310720 tid 2 2014-07-01 16:51:02,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: writing mira0435601-427 from 1310720 to 1863680 tid 3 2014-07-01 16:51:02,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: writing mira0435601-427 from 1863680 to 2146304 tid 4 2014-07-01 16:51:02,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: read oid 309 snap -1 2014-07-01 16:51:02,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2275: delete oid 10 current snap is 0 2014-07-01 16:51:02,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: expect (ObjNum 308 snap 0 seq_num 308) 2014-07-01 16:51:02,655.655 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:03,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 1 to mira0435601-427 2014-07-01 16:51:03,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 2 to mira0435601-427 2014-07-01 16:51:03,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 3 to mira0435601-427 2014-07-01 16:51:03,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 4 to mira0435601-427 2014-07-01 16:51:03,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 6 to mira0435601-427 2014-07-01 16:51:03,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: done (2 left) 2014-07-01 16:51:03,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2275: done (1 left) 2014-07-01 16:51:03,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2276: read oid 62 snap -1 2014-07-01 16:51:03,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 416 current snap is 0 2014-07-01 16:51:03,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: seq_num 1466 ranges {3563520=335872} 2014-07-01 16:51:03,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: writing mira0435601-416 from 3563520 to 3899392 tid 1 2014-07-01 16:51:03,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 75 current snap is 0 2014-07-01 16:51:03,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: seq_num 1467 ranges {4694016=761856,5455872=765952,6221824=36864} 2014-07-01 16:51:03,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0435601-75 from 4694016 to 5455872 tid 1 2014-07-01 16:51:03,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0435601-75 from 5455872 to 6221824 tid 2 2014-07-01 16:51:03,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0435601-75 from 6221824 to 6258688 tid 3 2014-07-01 16:51:03,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: read oid 214 snap -1 2014-07-01 16:51:03,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: finishing write tid 7 to mira0435601-427 2014-07-01 16:51:03,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 138 (ObjNum 1464 snap 0 seq_num 1464) dirty exists 2014-07-01 16:51:03,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: done (4 left) 2014-07-01 16:51:03,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2280: read oid 268 snap -1 2014-07-01 16:51:03,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2281: read oid 65 snap -1 2014-07-01 16:51:03,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 42 current snap is 0 2014-07-01 16:51:03,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: seq_num 1468 ranges {2551808=389120} 2014-07-01 16:51:03,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: writing mira0435601-42 from 2551808 to 2940928 tid 1 2014-07-01 16:51:03,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 447 current snap is 0 2014-07-01 16:51:03,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: seq_num 1469 ranges {3477504=507904,3985408=581632,4567040=221184} 2014-07-01 16:51:03,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: writing mira0435601-447 from 3477504 to 3985408 tid 1 2014-07-01 16:51:03,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: writing mira0435601-447 from 3985408 to 4567040 tid 2 2014-07-01 16:51:03,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: writing mira0435601-447 from 4567040 to 4788224 tid 3 2014-07-01 16:51:03,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 16:51:03,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: seq_num 1470 ranges {6631424=581632,7213056=401408} 2014-07-01 16:51:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: writing mira0435601-252 from 6631424 to 7213056 tid 1 2014-07-01 16:51:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: writing mira0435601-252 from 7213056 to 7614464 tid 2 2014-07-01 16:51:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 162 current snap is 0 2014-07-01 16:51:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: seq_num 1471 ranges {0=610304,610304=749568,1359872=774144,2134016=81920} 2014-07-01 16:51:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: writing mira0435601-162 from 0 to 610304 tid 1 2014-07-01 16:51:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: writing mira0435601-162 from 610304 to 1359872 tid 2 2014-07-01 16:51:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: writing mira0435601-162 from 1359872 to 2134016 tid 3 2014-07-01 16:51:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: writing mira0435601-162 from 2134016 to 2215936 tid 4 2014-07-01 16:51:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 167 current snap is 0 2014-07-01 16:51:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: seq_num 1472 ranges {0=724992} 2014-07-01 16:51:03,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: writing mira0435601-167 from 0 to 724992 tid 1 2014-07-01 16:51:03,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 319 current snap is 0 2014-07-01 16:51:03,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: seq_num 1473 ranges {73728=753664,827392=561152,1388544=643072} 2014-07-01 16:51:03,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: writing mira0435601-319 from 73728 to 827392 tid 1 2014-07-01 16:51:03,440.440 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:03,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: writing mira0435601-319 from 827392 to 1388544 tid 2 2014-07-01 16:51:03,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: writing mira0435601-319 from 1388544 to 2031616 tid 3 2014-07-01 16:51:03,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2288: read oid 359 snap -1 2014-07-01 16:51:03,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 463 current snap is 0 2014-07-01 16:51:03,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: seq_num 1474 ranges {1732608=782336} 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: writing mira0435601-463 from 1732608 to 2514944 tid 1 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 285 current snap is 0 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: seq_num 1475 ranges {0=143360} 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: writing mira0435601-285 from 0 to 143360 tid 1 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 16:51:03,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: seq_num 1476 ranges {4075520=573440,4648960=557056,5206016=245760} 2014-07-01 16:51:03,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0435601-492 from 4075520 to 4648960 tid 1 2014-07-01 16:51:03,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0435601-492 from 4648960 to 5206016 tid 2 2014-07-01 16:51:03,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0435601-492 from 5206016 to 5451776 tid 3 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: finishing write tid 1 to mira0435601-447 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: finishing write tid 2 to mira0435601-447 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: finishing write tid 3 to mira0435601-447 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: finishing write tid 5 to mira0435601-447 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: finishing write tid 6 to mira0435601-447 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 257 (ObjNum 1469 snap 0 seq_num 1469) dirty exists 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 1 to mira0435601-252 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 2 to mira0435601-252 2014-07-01 16:51:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 4 to mira0435601-252 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 5 to mira0435601-252 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 328 (ObjNum 1470 snap 0 seq_num 1470) dirty exists 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: done (15 left) 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: done (14 left) 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 16:51:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: seq_num 1477 ranges {6582272=200704} 2014-07-01 16:51:03,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: writing mira0435601-400 from 6582272 to 6782976 tid 1 2014-07-01 16:51:03,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:51:03,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: seq_num 1478 ranges {2023424=69632} 2014-07-01 16:51:03,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: writing mira0435601-280 from 2023424 to 2093056 tid 1 2014-07-01 16:51:03,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: finishing write tid 1 to mira0435601-400 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: finishing write tid 3 to mira0435601-400 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: finishing write tid 4 to mira0435601-400 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 243 (ObjNum 1477 snap 0 seq_num 1477) dirty exists 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: done (15 left) 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 436 current snap is 0 2014-07-01 16:51:03,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: seq_num 1479 ranges {3313664=794624} 2014-07-01 16:51:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: writing mira0435601-436 from 3313664 to 4108288 tid 1 2014-07-01 16:51:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:51:04,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2276: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:51:05,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: finishing write tid 1 to mira0435601-416 2014-07-01 16:51:05,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 1 to mira0435601-75 2014-07-01 16:51:05,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: finishing write tid 3 to mira0435601-416 2014-07-01 16:51:05,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 2 to mira0435601-75 2014-07-01 16:51:05,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: finishing write tid 1 to mira0435601-167 2014-07-01 16:51:05,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: expect (ObjNum 1441 snap 0 seq_num 1441) 2014-07-01 16:51:06,750.750 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:07,450.450 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:50:44.721894', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882745560, u'health': u'HEALTH_OK', u'kb_used': 14035492, u'store_stats': {u'bytes_total': 6707415, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:50:54.539292', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886565492, u'health': u'HEALTH_OK', u'kb_used': 10215560, u'store_stats': {u'bytes_total': 8804044, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:50:11.321898', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886585328, u'health': u'HEALTH_OK', u'kb_used': 10195724, u'store_stats': {u'bytes_total': 5658376, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'10 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'18 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'28 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 224/8898 objects degraded (2.517%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 16647622, u'degraded_objects': 224, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 16431850982, u'degraded_total': 8898, u'bytes_used': 33668886528, u'recovering_objects_per_sec': 3, u'write_bytes_sec': 2914654, u'op_per_sec': 43, u'version': 1664, u'pgs_by_state': [{u'count': 18, u'state_name': u'active+recovery_wait'}, {u'count': 360, u'state_name': u'active+clean'}, {u'count': 10, u'state_name': u'active+recovering'}], u'read_bytes_sec': 6494848, u'degraded_ratio': u'2.517', u'bytes_avail': 4964913651712}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:51:07,451.451 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:08,246.246 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:51:08,246.246 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:09,101.101 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:09,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2280: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:51:10,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 3 to mira0435601-75 2014-07-01 16:51:10,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: finishing write tid 3 to mira0435601-167 2014-07-01 16:51:10,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: finishing write tid 4 to mira0435601-416 2014-07-01 16:51:10,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 207 (ObjNum 1466 snap 0 seq_num 1466) dirty exists 2014-07-01 16:51:10,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: finishing write tid 4 to mira0435601-167 2014-07-01 16:51:10,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 167 v 136 (ObjNum 1472 snap 0 seq_num 1472) dirty exists 2014-07-01 16:51:10,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 5 to mira0435601-75 2014-07-01 16:51:10,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: finishing write tid 1 to mira0435601-463 2014-07-01 16:51:10,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2281: expect (ObjNum 937 snap 0 seq_num 937) 2014-07-01 16:51:11,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: finishing write tid 3 to mira0435601-463 2014-07-01 16:51:11,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: finishing write tid 4 to mira0435601-463 2014-07-01 16:51:11,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 127 (ObjNum 1474 snap 0 seq_num 1474) dirty exists 2014-07-01 16:51:11,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: finishing write tid 1 to mira0435601-280 2014-07-01 16:51:11,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: finishing write tid 3 to mira0435601-280 2014-07-01 16:51:11,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 1 to mira0435601-162 2014-07-01 16:51:11,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 2 to mira0435601-162 2014-07-01 16:51:11,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2276: done (15 left) 2014-07-01 16:51:11,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: done (14 left) 2014-07-01 16:51:11,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: done (13 left) 2014-07-01 16:51:11,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2280: done (12 left) 2014-07-01 16:51:11,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2281: done (11 left) 2014-07-01 16:51:11,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: done (10 left) 2014-07-01 16:51:11,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: done (9 left) 2014-07-01 16:51:11,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 64 current snap is 0 2014-07-01 16:51:11,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: seq_num 1480 ranges {2629632=667648} 2014-07-01 16:51:11,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: writing mira0435601-64 from 2629632 to 3297280 tid 1 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 29 current snap is 0 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: seq_num 1481 ranges {1609728=692224,2301952=770048,3072000=434176} 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: writing mira0435601-29 from 1609728 to 2301952 tid 1 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: writing mira0435601-29 from 2301952 to 3072000 tid 2 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: writing mira0435601-29 from 3072000 to 3506176 tid 3 2014-07-01 16:51:11,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2297: read oid 251 snap -1 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 3 to mira0435601-162 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 4 to mira0435601-162 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 6 to mira0435601-162 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 6 to mira0435601-75 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 267 (ObjNum 1467 snap 0 seq_num 1467) dirty exists 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: finishing write tid 7 to mira0435601-162 2014-07-01 16:51:11,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 219 (ObjNum 1471 snap 0 seq_num 1471) dirty exists 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: finishing write tid 4 to mira0435601-280 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 279 (ObjNum 1478 snap 0 seq_num 1478) dirty exists 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: finishing write tid 1 to mira0435601-42 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: finishing write tid 1 to mira0435601-319 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: finishing write tid 3 to mira0435601-42 2014-07-01 16:51:11,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: finishing write tid 4 to mira0435601-42 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 199 (ObjNum 1468 snap 0 seq_num 1468) dirty exists 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: finishing write tid 2 to mira0435601-319 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: finishing write tid 3 to mira0435601-319 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: finishing write tid 5 to mira0435601-319 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: finishing write tid 6 to mira0435601-319 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 163 (ObjNum 1473 snap 0 seq_num 1473) dirty exists 2014-07-01 16:51:11,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: finishing write tid 1 to mira0435601-285 2014-07-01 16:51:11,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:2288: expect (ObjNum 1289 snap 0 seq_num 1289) 2014-07-01 16:51:15,135.135 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:15,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: finishing write tid 3 to mira0435601-285 2014-07-01 16:51:15,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 1 to mira0435601-492 2014-07-01 16:51:15,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: finishing write tid 4 to mira0435601-285 2014-07-01 16:51:15,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 124 (ObjNum 1475 snap 0 seq_num 1475) dirty exists 2014-07-01 16:51:15,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 2 to mira0435601-492 2014-07-01 16:51:15,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 3 to mira0435601-492 2014-07-01 16:51:15,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 5 to mira0435601-492 2014-07-01 16:51:15,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 1 to mira0435601-436 2014-07-01 16:51:15,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 3 to mira0435601-436 2014-07-01 16:51:15,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 4 to mira0435601-436 2014-07-01 16:51:15,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 203 (ObjNum 1479 snap 0 seq_num 1479) dirty exists 2014-07-01 16:51:15,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 6 to mira0435601-492 2014-07-01 16:51:15,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 168 (ObjNum 1476 snap 0 seq_num 1476) dirty exists 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: finishing write tid 1 to mira0435601-64 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: finishing write tid 3 to mira0435601-64 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: finishing write tid 1 to mira0435601-29 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: finishing write tid 4 to mira0435601-64 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 190 (ObjNum 1480 snap 0 seq_num 1480) dirty exists 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: finishing write tid 2 to mira0435601-29 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: finishing write tid 3 to mira0435601-29 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: done (11 left) 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: done (10 left) 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: done (9 left) 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: done (8 left) 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2288: done (7 left) 2014-07-01 16:51:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: done (6 left) 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: done (5 left) 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: done (4 left) 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: done (3 left) 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: done (2 left) 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:51:15,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: seq_num 1482 ranges {4849664=565248,5414912=753664,6168576=450560} 2014-07-01 16:51:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: writing mira0435601-172 from 4849664 to 5414912 tid 1 2014-07-01 16:51:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: writing mira0435601-172 from 5414912 to 6168576 tid 2 2014-07-01 16:51:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: writing mira0435601-172 from 6168576 to 6619136 tid 3 2014-07-01 16:51:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2299: delete oid 424 current snap is 0 2014-07-01 16:51:15,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2297: expect (ObjNum 1334 snap 0 seq_num 1334) 2014-07-01 16:51:15,905.905 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:18,506.506 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:18,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: finishing write tid 5 to mira0435601-29 2014-07-01 16:51:18,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: finishing write tid 6 to mira0435601-29 2014-07-01 16:51:18,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 203 (ObjNum 1481 snap 0 seq_num 1481) dirty exists 2014-07-01 16:51:18,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: finishing write tid 1 to mira0435601-172 2014-07-01 16:51:18,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: finishing write tid 2 to mira0435601-172 2014-07-01 16:51:18,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: finishing write tid 3 to mira0435601-172 2014-07-01 16:51:18,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: finishing write tid 5 to mira0435601-172 2014-07-01 16:51:18,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: finishing write tid 6 to mira0435601-172 2014-07-01 16:51:18,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 211 (ObjNum 1482 snap 0 seq_num 1482) dirty exists 2014-07-01 16:51:18,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: done (3 left) 2014-07-01 16:51:18,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2297: done (2 left) 2014-07-01 16:51:18,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: done (1 left) 2014-07-01 16:51:18,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2299: done (0 left) 2014-07-01 16:51:18,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 140 current snap is 0 2014-07-01 16:51:18,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: seq_num 1484 ranges {5218304=512000,5730304=520192,6250496=282624} 2014-07-01 16:51:19,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: writing mira0435601-140 from 5218304 to 5730304 tid 1 2014-07-01 16:51:19,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: writing mira0435601-140 from 5730304 to 6250496 tid 2 2014-07-01 16:51:19,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: writing mira0435601-140 from 6250496 to 6533120 tid 3 2014-07-01 16:51:19,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 466 current snap is 0 2014-07-01 16:51:19,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: seq_num 1485 ranges {897024=786432,1683456=401408} 2014-07-01 16:51:19,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: writing mira0435601-466 from 897024 to 1683456 tid 1 2014-07-01 16:51:19,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: writing mira0435601-466 from 1683456 to 2084864 tid 2 2014-07-01 16:51:19,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2302: delete oid 335 current snap is 0 2014-07-01 16:51:19,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: finishing write tid 1 to mira0435601-140 2014-07-01 16:51:19,218.218 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:50:44.721894', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882745560, u'health': u'HEALTH_OK', u'kb_used': 14035492, u'store_stats': {u'bytes_total': 6707415, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:50:54.539292', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886565492, u'health': u'HEALTH_OK', u'kb_used': 10215560, u'store_stats': {u'bytes_total': 8804044, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:51:11.322329', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886556160, u'health': u'HEALTH_OK', u'kb_used': 10224892, u'store_stats': {u'bytes_total': 9852680, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'6 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'17 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'23 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 98/8683 objects degraded (1.129%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 23664670, u'degraded_objects': 98, u'num_pgs': 388, u'recovering_keys_per_sec': 0, u'data_bytes': 15776378880, u'degraded_total': 8683, u'bytes_used': 32687161344, u'recovering_objects_per_sec': 5, u'write_bytes_sec': 1295181, u'op_per_sec': 45, u'version': 1673, u'pgs_by_state': [{u'count': 17, u'state_name': u'active+recovery_wait'}, {u'count': 365, u'state_name': u'active+clean'}, {u'count': 6, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2039, u'degraded_ratio': u'1.129', u'bytes_avail': 4965895376896}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:51:19,219.219 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:19,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: finishing write tid 2 to mira0435601-140 2014-07-01 16:51:19,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: finishing write tid 3 to mira0435601-140 2014-07-01 16:51:19,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: finishing write tid 5 to mira0435601-140 2014-07-01 16:51:19,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: finishing write tid 1 to mira0435601-466 2014-07-01 16:51:19,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: finishing write tid 2 to mira0435601-466 2014-07-01 16:51:19,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: finishing write tid 4 to mira0435601-466 2014-07-01 16:51:19,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: finishing write tid 6 to mira0435601-140 2014-07-01 16:51:19,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 140 (ObjNum 1484 snap 0 seq_num 1484) dirty exists 2014-07-01 16:51:20,001.001 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:51:20,001.001 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:20,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: finishing write tid 5 to mira0435601-466 2014-07-01 16:51:20,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 194 (ObjNum 1485 snap 0 seq_num 1485) dirty exists 2014-07-01 16:51:20,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: done (2 left) 2014-07-01 16:51:20,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: done (1 left) 2014-07-01 16:51:20,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2302: done (0 left) 2014-07-01 16:51:20,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:2303: delete oid 5 current snap is 0 2014-07-01 16:51:20,832.832 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:20,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2303: done (0 left) 2014-07-01 16:51:20,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 451 current snap is 0 2014-07-01 16:51:20,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: seq_num 1488 ranges {827392=606208} 2014-07-01 16:51:20,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: writing mira0435601-451 from 827392 to 1433600 tid 1 2014-07-01 16:51:20,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 364 current snap is 0 2014-07-01 16:51:20,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: seq_num 1489 ranges {2531328=479232} 2014-07-01 16:51:20,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: writing mira0435601-364 from 2531328 to 3010560 tid 1 2014-07-01 16:51:20,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 282 current snap is 0 2014-07-01 16:51:20,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: seq_num 1490 ranges {3866624=503808,4370432=561152,4931584=643072} 2014-07-01 16:51:21,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: writing mira0435601-282 from 3866624 to 4370432 tid 1 2014-07-01 16:51:21,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: writing mira0435601-282 from 4370432 to 4931584 tid 2 2014-07-01 16:51:21,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: writing mira0435601-282 from 4931584 to 5574656 tid 3 2014-07-01 16:51:21,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: read oid 234 snap -1 2014-07-01 16:51:21,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 485 current snap is 0 2014-07-01 16:51:21,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: seq_num 1491 ranges {0=532480} 2014-07-01 16:51:21,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: writing mira0435601-485 from 0 to 532480 tid 1 2014-07-01 16:51:21,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: read oid 406 snap -1 2014-07-01 16:51:21,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: read oid 339 snap -1 2014-07-01 16:51:21,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 16:51:21,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: seq_num 1492 ranges {3776512=651264,4427776=724992,5152768=77824} 2014-07-01 16:51:21,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: writing mira0435601-226 from 3776512 to 4427776 tid 1 2014-07-01 16:51:21,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: writing mira0435601-226 from 4427776 to 5152768 tid 2 2014-07-01 16:51:21,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: writing mira0435601-226 from 5152768 to 5230592 tid 3 2014-07-01 16:51:21,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:51:21,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: seq_num 1493 ranges {6619136=724992,7344128=401408} 2014-07-01 16:51:21,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: writing mira0435601-172 from 6619136 to 7344128 tid 1 2014-07-01 16:51:21,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: writing mira0435601-172 from 7344128 to 7745536 tid 2 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2313: read oid 198 snap -1 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: finishing write tid 1 to mira0435601-451 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: finishing write tid 3 to mira0435601-451 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: finishing write tid 1 to mira0435601-282 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: finishing write tid 2 to mira0435601-282 2014-07-01 16:51:21,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: finishing write tid 3 to mira0435601-282 2014-07-01 16:51:21,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: finishing write tid 4 to mira0435601-451 2014-07-01 16:51:21,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 252 (ObjNum 1488 snap 0 seq_num 1488) dirty exists 2014-07-01 16:51:21,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: expect (ObjNum 980 snap 0 seq_num 980) 2014-07-01 16:51:23,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: finishing write tid 5 to mira0435601-282 2014-07-01 16:51:23,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: expect (ObjNum 1402 snap 0 seq_num 1402) 2014-07-01 16:51:23,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 1 to mira0435601-364 2014-07-01 16:51:23,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: finishing write tid 1 to mira0435601-226 2014-07-01 16:51:23,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: finishing write tid 1 to mira0435601-485 2014-07-01 16:51:23,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: finishing write tid 2 to mira0435601-226 2014-07-01 16:51:23,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: done (9 left) 2014-07-01 16:51:23,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: done (8 left) 2014-07-01 16:51:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: done (7 left) 2014-07-01 16:51:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: done (6 left) 2014-07-01 16:51:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: read oid 248 snap -1 2014-07-01 16:51:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: finishing write tid 3 to mira0435601-485 2014-07-01 16:51:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 3 to mira0435601-364 2014-07-01 16:51:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: finishing write tid 6 to mira0435601-282 2014-07-01 16:51:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 256 (ObjNum 1490 snap 0 seq_num 1490) dirty exists 2014-07-01 16:51:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:2313: expect (ObjNum 1189 snap 0 seq_num 1189) 2014-07-01 16:51:25,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: finishing write tid 3 to mira0435601-226 2014-07-01 16:51:25,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: finishing write tid 4 to mira0435601-485 2014-07-01 16:51:25,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 245 (ObjNum 1491 snap 0 seq_num 1491) dirty exists 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: finishing write tid 5 to mira0435601-226 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: finishing write tid 6 to mira0435601-226 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 116 (ObjNum 1492 snap 0 seq_num 1492) dirty exists 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 4 to mira0435601-364 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 364 v 170 (ObjNum 1489 snap 0 seq_num 1489) dirty exists 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: finishing write tid 1 to mira0435601-172 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: finishing write tid 2 to mira0435601-172 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: finishing write tid 4 to mira0435601-172 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: done (6 left) 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: done (5 left) 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: done (4 left) 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: done (3 left) 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2313: done (2 left) 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 241 current snap is 0 2014-07-01 16:51:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: seq_num 1494 ranges {3616768=753664,4370432=540672,4911104=782336,5693440=282624} 2014-07-01 16:51:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: writing mira0435601-241 from 3616768 to 4370432 tid 1 2014-07-01 16:51:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: writing mira0435601-241 from 4370432 to 4911104 tid 2 2014-07-01 16:51:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: writing mira0435601-241 from 4911104 to 5693440 tid 3 2014-07-01 16:51:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: writing mira0435601-241 from 5693440 to 5976064 tid 4 2014-07-01 16:51:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 343 current snap is 0 2014-07-01 16:51:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: seq_num 1495 ranges {0=466944,466944=401408} 2014-07-01 16:51:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: writing mira0435601-343 from 0 to 466944 tid 1 2014-07-01 16:51:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: writing mira0435601-343 from 466944 to 868352 tid 2 2014-07-01 16:51:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: delete oid 417 current snap is 0 2014-07-01 16:51:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: finishing write tid 5 to mira0435601-172 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 214 (ObjNum 1493 snap 0 seq_num 1493) dirty exists 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: done (4 left) 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: done (3 left) 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: done (2 left) 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 12 current snap is 0 2014-07-01 16:51:25,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: seq_num 1497 ranges {880640=573440,1454080=782336,2236416=421888} 2014-07-01 16:51:25,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: writing mira0435601-12 from 880640 to 1454080 tid 1 2014-07-01 16:51:25,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: writing mira0435601-12 from 1454080 to 2236416 tid 2 2014-07-01 16:51:25,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: writing mira0435601-12 from 2236416 to 2658304 tid 3 2014-07-01 16:51:25,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: delete oid 433 current snap is 0 2014-07-01 16:51:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 1 to mira0435601-241 2014-07-01 16:51:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 1 to mira0435601-343 2014-07-01 16:51:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 2 to mira0435601-241 2014-07-01 16:51:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 2 to mira0435601-343 2014-07-01 16:51:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 3 to mira0435601-241 2014-07-01 16:51:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 4 to mira0435601-241 2014-07-01 16:51:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 4 to mira0435601-343 2014-07-01 16:51:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 6 to mira0435601-241 2014-07-01 16:51:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: finishing write tid 1 to mira0435601-12 2014-07-01 16:51:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 7 to mira0435601-241 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 210 (ObjNum 1494 snap 0 seq_num 1494) dirty exists 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 5 to mira0435601-343 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 240 (ObjNum 1495 snap 0 seq_num 1495) dirty exists 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: finishing write tid 2 to mira0435601-12 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: finishing write tid 3 to mira0435601-12 2014-07-01 16:51:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: finishing write tid 5 to mira0435601-12 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: done (3 left) 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: done (2 left) 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: done (1 left) 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 235 current snap is 0 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: seq_num 1499 ranges {3674112=716800,4390912=548864,4939776=253952} 2014-07-01 16:51:25,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: writing mira0435601-235 from 3674112 to 4390912 tid 1 2014-07-01 16:51:25,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: writing mira0435601-235 from 4390912 to 4939776 tid 2 2014-07-01 16:51:25,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: writing mira0435601-235 from 4939776 to 5193728 tid 3 2014-07-01 16:51:25,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:2321: read oid 402 snap -1 2014-07-01 16:51:25,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: finishing write tid 6 to mira0435601-12 2014-07-01 16:51:25,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 102 (ObjNum 1497 snap 0 seq_num 1497) dirty exists 2014-07-01 16:51:25,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: done (2 left) 2014-07-01 16:51:25,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:2322: delete oid 296 current snap is 0 2014-07-01 16:51:25,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: finishing write tid 1 to mira0435601-235 2014-07-01 16:51:25,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2322: done (2 left) 2014-07-01 16:51:25,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2323: read oid 331 snap -1 2014-07-01 16:51:25,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2324: delete oid 447 current snap is 0 2014-07-01 16:51:25,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: finishing write tid 2 to mira0435601-235 2014-07-01 16:51:25,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2323: done (3 left) 2014-07-01 16:51:25,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2324: done (2 left) 2014-07-01 16:51:25,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 16:51:25,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: seq_num 1502 ranges {4345856=536576,4882432=401408} 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: writing mira0435601-434 from 4345856 to 4882432 tid 1 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: writing mira0435601-434 from 4882432 to 5283840 tid 2 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: seq_num 1503 ranges {2093056=565248,2658304=528384,3186688=548864,3735552=528384} 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: writing mira0435601-280 from 2093056 to 2658304 tid 1 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: writing mira0435601-280 from 2658304 to 3186688 tid 2 2014-07-01 16:51:25,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: writing mira0435601-280 from 3186688 to 3735552 tid 3 2014-07-01 16:51:25,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: writing mira0435601-280 from 3735552 to 4263936 tid 4 2014-07-01 16:51:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: read oid 62 snap -1 2014-07-01 16:51:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: finishing write tid 1 to mira0435601-434 2014-07-01 16:51:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: finishing write tid 2 to mira0435601-434 2014-07-01 16:51:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: finishing write tid 4 to mira0435601-434 2014-07-01 16:51:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: finishing write tid 3 to mira0435601-235 2014-07-01 16:51:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: finishing write tid 5 to mira0435601-235 2014-07-01 16:51:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 188 current snap is 0 2014-07-01 16:51:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: seq_num 1504 ranges {0=434176,434176=512000,946176=622592,1568768=454656,2023424=16384} 2014-07-01 16:51:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0435601-188 from 0 to 434176 tid 1 2014-07-01 16:51:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0435601-188 from 434176 to 946176 tid 2 2014-07-01 16:51:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0435601-188 from 946176 to 1568768 tid 3 2014-07-01 16:51:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0435601-188 from 1568768 to 2023424 tid 4 2014-07-01 16:51:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0435601-188 from 2023424 to 2039808 tid 5 2014-07-01 16:51:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 441 current snap is 0 2014-07-01 16:51:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: seq_num 1505 ranges {1720320=708608,2428928=745472,3174400=458752} 2014-07-01 16:51:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: writing mira0435601-441 from 1720320 to 2428928 tid 1 2014-07-01 16:51:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: writing mira0435601-441 from 2428928 to 3174400 tid 2 2014-07-01 16:51:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: writing mira0435601-441 from 3174400 to 3633152 tid 3 2014-07-01 16:51:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2330: read oid 210 snap -1 2014-07-01 16:51:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: finishing write tid 5 to mira0435601-434 2014-07-01 16:51:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 202 (ObjNum 1502 snap 0 seq_num 1502) dirty exists 2014-07-01 16:51:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: finishing write tid 1 to mira0435601-441 2014-07-01 16:51:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: done (7 left) 2014-07-01 16:51:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 73 current snap is 0 2014-07-01 16:51:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: seq_num 1506 ranges {3710976=786432,4497408=729088,5226496=73728} 2014-07-01 16:51:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: writing mira0435601-73 from 3710976 to 4497408 tid 1 2014-07-01 16:51:25,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: writing mira0435601-73 from 4497408 to 5226496 tid 2 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: writing mira0435601-73 from 5226496 to 5300224 tid 3 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2332: delete oid 112 current snap is 0 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: finishing write tid 2 to mira0435601-441 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: finishing write tid 3 to mira0435601-441 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: finishing write tid 5 to mira0435601-441 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: finishing write tid 6 to mira0435601-441 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 234 (ObjNum 1505 snap 0 seq_num 1505) dirty exists 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: done (8 left) 2014-07-01 16:51:25,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2332: done (7 left) 2014-07-01 16:51:25,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 469 current snap is 0 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: seq_num 1508 ranges {3665920=528384,4194304=741376,4935680=61440} 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: writing mira0435601-469 from 3665920 to 4194304 tid 1 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: writing mira0435601-469 from 4194304 to 4935680 tid 2 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: writing mira0435601-469 from 4935680 to 4997120 tid 3 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 83 current snap is 0 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: seq_num 1509 ranges {1904640=155648} 2014-07-01 16:51:25,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: writing mira0435601-83 from 1904640 to 2060288 tid 1 2014-07-01 16:51:25,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: delete oid 324 current snap is 0 2014-07-01 16:51:25,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2330: expect (ObjNum 1456 snap 0 seq_num 1456) 2014-07-01 16:51:26,413.413 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:27,185.185 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: finishing write tid 1 to mira0435601-469 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: finishing write tid 2 to mira0435601-469 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: finishing write tid 3 to mira0435601-469 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: finishing write tid 5 to mira0435601-469 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: finishing write tid 6 to mira0435601-469 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 120 (ObjNum 1508 snap 0 seq_num 1508) dirty exists 2014-07-01 16:51:27,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2321: expect (ObjNum 401 snap 0 seq_num 401) 2014-07-01 16:51:29,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 1 to mira0435601-280 2014-07-01 16:51:29,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 2 to mira0435601-280 2014-07-01 16:51:29,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2321: done (9 left) 2014-07-01 16:51:29,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2330: done (8 left) 2014-07-01 16:51:29,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: done (7 left) 2014-07-01 16:51:29,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: done (6 left) 2014-07-01 16:51:29,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:2336: read oid 22 snap -1 2014-07-01 16:51:29,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 3 to mira0435601-280 2014-07-01 16:51:29,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:51:29,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 1 to mira0435601-188 2014-07-01 16:51:29,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: done (6 left) 2014-07-01 16:51:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: read oid 41 snap -1 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 2 to mira0435601-188 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 3 to mira0435601-188 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: finishing write tid 6 to mira0435601-235 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 194 (ObjNum 1499 snap 0 seq_num 1499) dirty exists 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 4 to mira0435601-188 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 4 to mira0435601-280 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 5 to mira0435601-188 2014-07-01 16:51:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 6 to mira0435601-280 2014-07-01 16:51:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 7 to mira0435601-188 2014-07-01 16:51:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: finishing write tid 7 to mira0435601-280 2014-07-01 16:51:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 284 (ObjNum 1503 snap 0 seq_num 1503) dirty exists 2014-07-01 16:51:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: finishing write tid 1 to mira0435601-73 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: finishing write tid 2 to mira0435601-73 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: done (6 left) 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: done (5 left) 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: read oid 10 snap -1 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: finishing write tid 3 to mira0435601-73 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: finishing write tid 5 to mira0435601-73 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: finishing write tid 1 to mira0435601-83 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: finishing write tid 3 to mira0435601-83 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: finishing write tid 6 to mira0435601-73 2014-07-01 16:51:29,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 223 (ObjNum 1506 snap 0 seq_num 1506) dirty exists 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: done (5 left) 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 295 current snap is 0 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: seq_num 1511 ranges {2805760=749568} 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: writing mira0435601-295 from 2805760 to 3555328 tid 1 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 381 current snap is 0 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: seq_num 1512 ranges {0=778240,778240=720896,1499136=483328} 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0435601-381 from 0 to 778240 tid 1 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0435601-381 from 778240 to 1499136 tid 2 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0435601-381 from 1499136 to 1982464 tid 3 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 126 current snap is 0 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: seq_num 1513 ranges {0=647168,647168=548864,1196032=634880,1830912=20480} 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: writing mira0435601-126 from 0 to 647168 tid 1 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: writing mira0435601-126 from 647168 to 1196032 tid 2 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: writing mira0435601-126 from 1196032 to 1830912 tid 3 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: writing mira0435601-126 from 1830912 to 1851392 tid 4 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 120 current snap is 0 2014-07-01 16:51:29,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: seq_num 1514 ranges {3592192=520192,4112384=532480,4644864=507904,5152768=163840} 2014-07-01 16:51:29,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: writing mira0435601-120 from 3592192 to 4112384 tid 1 2014-07-01 16:51:29,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: writing mira0435601-120 from 4112384 to 4644864 tid 2 2014-07-01 16:51:29,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: writing mira0435601-120 from 4644864 to 5152768 tid 3 2014-07-01 16:51:29,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: writing mira0435601-120 from 5152768 to 5316608 tid 4 2014-07-01 16:51:29,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 16:51:29,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: seq_num 1515 ranges {1708032=593920,2301952=716800,3018752=86016} 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0435601-48 from 1708032 to 2301952 tid 1 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0435601-48 from 2301952 to 3018752 tid 2 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0435601-48 from 3018752 to 3104768 tid 3 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2344: delete oid 156 current snap is 0 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: finishing write tid 4 to mira0435601-83 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 225 (ObjNum 1509 snap 0 seq_num 1509) dirty exists 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 8 to mira0435601-188 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 144 (ObjNum 1504 snap 0 seq_num 1504) dirty exists 2014-07-01 16:51:29,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2336: expect (ObjNum 638 snap 0 seq_num 638) 2014-07-01 16:51:30,767.767 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:31,489.489 INFO:teuthology.orchestra.run.mira043.stderr:testContainerSerializedInfo (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,531.531 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:50:44.721894', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882745560, u'health': u'HEALTH_OK', u'kb_used': 14035492, u'store_stats': {u'bytes_total': 6707415, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:50:54.539292', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886565492, u'health': u'HEALTH_OK', u'kb_used': 10215560, u'store_stats': {u'bytes_total': 8804044, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:51:11.322329', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886556160, u'health': u'HEALTH_OK', u'kb_used': 10224892, u'store_stats': {u'bytes_total': 9852680, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'6 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'6 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery -35/6802 objects degraded (-0.515%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 18446744073709551581L, u'num_pgs': 388, u'data_bytes': 11564857098, u'degraded_total': 6802, u'bytes_used': 27948404736, u'op_per_sec': 337, u'write_bytes_sec': 3576947, u'version': 1681, u'pgs_by_state': [{u'count': 382, u'state_name': u'active+clean'}, {u'count': 6, u'state_name': u'active+recovering'}], u'read_bytes_sec': 2024547, u'degraded_ratio': u'-0.515', u'bytes_avail': 4970634133504}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:51:31,532.532 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:31,657.657 INFO:teuthology.orchestra.run.mira043.stderr:testContainersOrderedByName (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,676.676 INFO:teuthology.orchestra.run.mira043.stderr:testInvalidAuthToken (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,824.824 INFO:teuthology.orchestra.run.mira043.stderr:testInvalidUTF8Path (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,824.824 INFO:teuthology.orchestra.run.mira043.stderr:testLastContainerMarker (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,825.825 INFO:teuthology.orchestra.run.mira043.stderr:testListingLimit (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:31,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: finishing write tid 1 to mira0435601-295 2014-07-01 16:51:31,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: finishing write tid 3 to mira0435601-295 2014-07-01 16:51:31,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: done (10 left) 2014-07-01 16:51:31,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: done (9 left) 2014-07-01 16:51:31,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2336: done (8 left) 2014-07-01 16:51:31,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: done (7 left) 2014-07-01 16:51:31,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2344: done (6 left) 2014-07-01 16:51:31,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2345: delete oid 218 current snap is 0 2014-07-01 16:51:31,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 1 to mira0435601-381 2014-07-01 16:51:31,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 2 to mira0435601-381 2014-07-01 16:51:32,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: expect (ObjNum 40 snap 0 seq_num 40) 2014-07-01 16:51:32,255.255 INFO:teuthology.orchestra.run.mira043.stderr:testMarkerLimitContainerList (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:32,256.256 INFO:teuthology.orchestra.run.mira043.stderr:testPUT (test.functional.tests.TestAccountUTF8) ... ok 2014-07-01 16:51:32,257.257 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:51:32,257.257 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 3 to mira0435601-381 2014-07-01 16:51:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 5 to mira0435601-381 2014-07-01 16:51:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 6 to mira0435601-381 2014-07-01 16:51:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 207 (ObjNum 1512 snap 0 seq_num 1512) dirty exists 2014-07-01 16:51:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 1 to mira0435601-126 2014-07-01 16:51:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 2 to mira0435601-126 2014-07-01 16:51:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 3 to mira0435601-126 2014-07-01 16:51:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 4 to mira0435601-126 2014-07-01 16:51:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 6 to mira0435601-126 2014-07-01 16:51:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: finishing write tid 7 to mira0435601-126 2014-07-01 16:51:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 245 (ObjNum 1513 snap 0 seq_num 1513) dirty exists 2014-07-01 16:51:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: finishing write tid 4 to mira0435601-295 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 258 (ObjNum 1511 snap 0 seq_num 1511) dirty exists 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 1 to mira0435601-120 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 2 to mira0435601-120 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 3 to mira0435601-120 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 4 to mira0435601-120 2014-07-01 16:51:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 6 to mira0435601-120 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: finishing write tid 7 to mira0435601-120 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 132 (ObjNum 1514 snap 0 seq_num 1514) dirty exists 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 1 to mira0435601-48 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 2 to mira0435601-48 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 3 to mira0435601-48 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 5 to mira0435601-48 2014-07-01 16:51:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 6 to mira0435601-48 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 330 (ObjNum 1515 snap 0 seq_num 1515) dirty exists 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: done (6 left) 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: done (5 left) 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: done (4 left) 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: done (3 left) 2014-07-01 16:51:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: done (2 left) 2014-07-01 16:51:32,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: done (1 left) 2014-07-01 16:51:32,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2345: done (0 left) 2014-07-01 16:51:32,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 464 current snap is 0 2014-07-01 16:51:32,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: seq_num 1518 ranges {5709824=765952,6475776=516096,6991872=507904,7499776=512000,8011776=69632} 2014-07-01 16:51:32,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: writing mira0435601-464 from 5709824 to 6475776 tid 1 2014-07-01 16:51:32,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: writing mira0435601-464 from 6475776 to 6991872 tid 2 2014-07-01 16:51:32,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: writing mira0435601-464 from 6991872 to 7499776 tid 3 2014-07-01 16:51:32,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: writing mira0435601-464 from 7499776 to 8011776 tid 4 2014-07-01 16:51:32,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: writing mira0435601-464 from 8011776 to 8081408 tid 5 2014-07-01 16:51:32,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 16:51:32,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: seq_num 1519 ranges {4091904=638976,4730880=540672,5271552=786432,6057984=278528} 2014-07-01 16:51:32,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: writing mira0435601-311 from 4091904 to 4730880 tid 1 2014-07-01 16:51:32,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: writing mira0435601-311 from 4730880 to 5271552 tid 2 2014-07-01 16:51:32,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: writing mira0435601-311 from 5271552 to 6057984 tid 3 2014-07-01 16:51:32,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: writing mira0435601-311 from 6057984 to 6336512 tid 4 2014-07-01 16:51:33,028.028 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:33,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 1 to mira0435601-464 2014-07-01 16:51:33,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 2 to mira0435601-464 2014-07-01 16:51:33,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 3 to mira0435601-464 2014-07-01 16:51:33,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 4 to mira0435601-464 2014-07-01 16:51:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 5 to mira0435601-464 2014-07-01 16:51:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 7 to mira0435601-464 2014-07-01 16:51:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: finishing write tid 8 to mira0435601-464 2014-07-01 16:51:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 334 (ObjNum 1518 snap 0 seq_num 1518) dirty exists 2014-07-01 16:51:33,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 1 to mira0435601-311 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 2 to mira0435601-311 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2348: read oid 488 snap -1 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: done (2 left) 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 357 current snap is 0 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: seq_num 1520 ranges {0=716800,716800=524288,1241088=454656,1695744=225280} 2014-07-01 16:51:33,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: writing mira0435601-357 from 0 to 716800 tid 1 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: writing mira0435601-357 from 716800 to 1241088 tid 2 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: writing mira0435601-357 from 1241088 to 1695744 tid 3 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: writing mira0435601-357 from 1695744 to 1921024 tid 4 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: read oid 87 snap -1 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 3 to mira0435601-311 2014-07-01 16:51:33,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 4 to mira0435601-311 2014-07-01 16:51:33,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 6 to mira0435601-311 2014-07-01 16:51:33,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: finishing write tid 7 to mira0435601-311 2014-07-01 16:51:33,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 240 (ObjNum 1519 snap 0 seq_num 1519) dirty exists 2014-07-01 16:51:33,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2348: expect (ObjNum 487 snap 0 seq_num 487) 2014-07-01 16:51:34,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 1 to mira0435601-357 2014-07-01 16:51:34,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 2 to mira0435601-357 2014-07-01 16:51:34,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: done (3 left) 2014-07-01 16:51:34,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2348: done (2 left) 2014-07-01 16:51:34,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: read oid 125 snap -1 2014-07-01 16:51:34,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 3 to mira0435601-357 2014-07-01 16:51:34,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 4 to mira0435601-357 2014-07-01 16:51:34,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 6 to mira0435601-357 2014-07-01 16:51:34,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: expect (ObjNum 1362 snap 0 seq_num 1362) 2014-07-01 16:51:34,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: finishing write tid 7 to mira0435601-357 2014-07-01 16:51:34,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 339 (ObjNum 1520 snap 0 seq_num 1520) dirty exists 2014-07-01 16:51:34,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: expect (ObjNum 1320 snap 0 seq_num 1320) 2014-07-01 16:51:35,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: done (2 left) 2014-07-01 16:51:35,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: done (1 left) 2014-07-01 16:51:35,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: done (0 left) 2014-07-01 16:51:35,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 443 current snap is 0 2014-07-01 16:51:35,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: seq_num 1521 ranges {2494464=430080} 2014-07-01 16:51:35,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: writing mira0435601-443 from 2494464 to 2924544 tid 1 2014-07-01 16:51:35,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: read oid 348 snap -1 2014-07-01 16:51:35,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: read oid 284 snap -1 2014-07-01 16:51:35,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2355: delete oid 89 current snap is 0 2014-07-01 16:51:35,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: expect (ObjNum 1283 snap 0 seq_num 1283) 2014-07-01 16:51:35,416.416 INFO:teuthology.orchestra.run.mira043.stderr:testContainerExistenceCachingProblem (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,463.463 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileList (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,463.463 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileListOnContainerThatDoesNotExist (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,463.463 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileListWithLimit (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,463.463 INFO:teuthology.orchestra.run.mira043.stderr:testContainerInfo (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,463.463 INFO:teuthology.orchestra.run.mira043.stderr:testContainerInfoOnContainerThatDoesNotExist (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:35,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 1 to mira0435601-443 2014-07-01 16:51:35,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 3 to mira0435601-443 2014-07-01 16:51:35,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 4 to mira0435601-443 2014-07-01 16:51:35,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 254 (ObjNum 1521 snap 0 seq_num 1521) dirty exists 2014-07-01 16:51:35,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: expect (ObjNum 1171 snap 0 seq_num 1171) 2014-07-01 16:51:35,983.983 INFO:teuthology.orchestra.run.mira043.stderr:testContainerNameLimit (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,013.013 INFO:teuthology.orchestra.run.mira043.stderr:testCreate (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,153.153 INFO:teuthology.orchestra.run.mira043.stderr:testDelete (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,153.153 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOnContainerThatDoesNotExist (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,154.154 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOnContainerWithFiles (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,154.154 INFO:teuthology.orchestra.run.mira043.stderr:testFileCreateInContainerThatDoesNotExist (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: done (3 left) 2014-07-01 16:51:36,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: done (2 left) 2014-07-01 16:51:36,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: done (1 left) 2014-07-01 16:51:36,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2355: done (0 left) 2014-07-01 16:51:36,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 374 current snap is 0 2014-07-01 16:51:36,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: seq_num 1523 ranges {3817472=487424} 2014-07-01 16:51:36,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: writing mira0435601-374 from 3817472 to 4304896 tid 1 2014-07-01 16:51:36,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: read oid 385 snap -1 2014-07-01 16:51:36,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2358: read oid 238 snap -1 2014-07-01 16:51:36,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2359: read oid 80 snap -1 2014-07-01 16:51:36,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 438 current snap is 0 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: seq_num 1524 ranges {2572288=651264,3223552=507904,3731456=503808,4235264=593920} 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: writing mira0435601-438 from 2572288 to 3223552 tid 1 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: writing mira0435601-438 from 3223552 to 3731456 tid 2 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: writing mira0435601-438 from 3731456 to 4235264 tid 3 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: writing mira0435601-438 from 4235264 to 4829184 tid 4 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 267 current snap is 0 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: seq_num 1525 ranges {2134016=679936,2813952=401408} 2014-07-01 16:51:36,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: writing mira0435601-267 from 2134016 to 2813952 tid 1 2014-07-01 16:51:36,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: writing mira0435601-267 from 2813952 to 3215360 tid 2 2014-07-01 16:51:36,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 466 current snap is 0 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: seq_num 1526 ranges {2084864=708608,2793472=720896,3514368=802816,4317184=81920} 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: writing mira0435601-466 from 2084864 to 2793472 tid 1 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: writing mira0435601-466 from 2793472 to 3514368 tid 2 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: writing mira0435601-466 from 3514368 to 4317184 tid 3 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: writing mira0435601-466 from 4317184 to 4399104 tid 4 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2363: read oid 310 snap -1 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 1 to mira0435601-374 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 3 to mira0435601-374 2014-07-01 16:51:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 4 to mira0435601-374 2014-07-01 16:51:36,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 115 (ObjNum 1523 snap 0 seq_num 1523) dirty exists 2014-07-01 16:51:36,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: expect (ObjNum 384 snap 0 seq_num 384) 2014-07-01 16:51:36,736.736 INFO:teuthology.orchestra.run.mira043.stderr:testFileListingLimitMarkerPrefix (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,737.737 INFO:teuthology.orchestra.run.mira043.stderr:testFileOrder (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,765.765 INFO:teuthology.orchestra.run.mira043.stderr:testFileThenContainerDelete (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,901.901 INFO:teuthology.orchestra.run.mira043.stderr:testLastFileMarker (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:36,904.904 INFO:teuthology.orchestra.run.mira043.stderr:testMarkerLimitFileList (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:37,367.367 INFO:teuthology.orchestra.run.mira043.stderr:testPrefixAndLimit (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:37,367.367 INFO:teuthology.orchestra.run.mira043.stderr:testTooLongName (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:37,533.533 INFO:teuthology.orchestra.run.mira043.stderr:testUtf8Container (test.functional.tests.TestContainer) ... ok 2014-07-01 16:51:38,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 1 to mira0435601-438 2014-07-01 16:51:38,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 2 to mira0435601-438 2014-07-01 16:51:38,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: done (7 left) 2014-07-01 16:51:38,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: done (6 left) 2014-07-01 16:51:38,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2358: done (5 left) 2014-07-01 16:51:38,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:51:38,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: seq_num 1527 ranges {0=577536,577536=503808,1081344=675840,1757184=425984} 2014-07-01 16:51:38,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0435601-445 from 0 to 577536 tid 1 2014-07-01 16:51:38,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0435601-445 from 577536 to 1081344 tid 2 2014-07-01 16:51:38,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0435601-445 from 1081344 to 1757184 tid 3 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0435601-445 from 1757184 to 2183168 tid 4 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 494 current snap is 0 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: seq_num 1528 ranges {2203648=450560,2654208=487424,3141632=503808,3645440=544768,4190208=69632} 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: writing mira0435601-494 from 2203648 to 2654208 tid 1 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: writing mira0435601-494 from 2654208 to 3141632 tid 2 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: writing mira0435601-494 from 3141632 to 3645440 tid 3 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: writing mira0435601-494 from 3645440 to 4190208 tid 4 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: writing mira0435601-494 from 4190208 to 4259840 tid 5 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 70 current snap is 0 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: seq_num 1529 ranges {393216=528384,921600=561152,1482752=622592,2105344=20480} 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: writing mira0435601-70 from 393216 to 921600 tid 1 2014-07-01 16:51:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: writing mira0435601-70 from 921600 to 1482752 tid 2 2014-07-01 16:51:38,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: writing mira0435601-70 from 1482752 to 2105344 tid 3 2014-07-01 16:51:38,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: writing mira0435601-70 from 2105344 to 2125824 tid 4 2014-07-01 16:51:38,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2367: read oid 94 snap -1 2014-07-01 16:51:38,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2359: expect (ObjNum 1052 snap 0 seq_num 1052) 2014-07-01 16:51:38,595.595 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:38,958.958 INFO:teuthology.orchestra.run.mira043.stderr:testContainerListing (test.functional.tests.TestContainerPaths) ... ok 2014-07-01 16:51:39,007.007 INFO:teuthology.orchestra.run.mira043.stderr:testStructure (test.functional.tests.TestContainerPaths) ... ok 2014-07-01 16:51:39,124.124 INFO:teuthology.orchestra.run.mira043.stderr:testTraverseContainer (test.functional.tests.TestContainerPaths) ... ok 2014-07-01 16:51:39,370.370 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:39,846.846 INFO:teuthology.orchestra.run.mira043.stderr:testContainerExistenceCachingProblem (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:39,861.861 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileList (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,017.017 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileListOnContainerThatDoesNotExist (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,018.018 INFO:teuthology.orchestra.run.mira043.stderr:testContainerFileListWithLimit (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,018.018 INFO:teuthology.orchestra.run.mira043.stderr:testContainerInfo (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,019.019 INFO:teuthology.orchestra.run.mira043.stderr:testContainerInfoOnContainerThatDoesNotExist (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,020.020 INFO:teuthology.orchestra.run.mira043.stderr:testContainerNameLimit (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,041.041 INFO:teuthology.orchestra.run.mira043.stderr:testCreate (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,092.092 INFO:teuthology.orchestra.run.mira043.stderr:testDelete (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,181.181 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOnContainerThatDoesNotExist (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,182.182 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOnContainerWithFiles (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,182.182 INFO:teuthology.orchestra.run.mira043.stderr:testFileCreateInContainerThatDoesNotExist (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,611.611 INFO:teuthology.orchestra.run.mira043.stderr:testFileListingLimitMarkerPrefix (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,624.624 INFO:teuthology.orchestra.run.mira043.stderr:testFileOrder (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 3 to mira0435601-438 2014-07-01 16:51:40,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 4 to mira0435601-438 2014-07-01 16:51:40,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 6 to mira0435601-438 2014-07-01 16:51:40,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: finishing write tid 7 to mira0435601-438 2014-07-01 16:51:40,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 212 (ObjNum 1524 snap 0 seq_num 1524) dirty exists 2014-07-01 16:51:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2359: done (8 left) 2014-07-01 16:51:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: done (7 left) 2014-07-01 16:51:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 464 current snap is 0 2014-07-01 16:51:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: seq_num 1530 ranges {8081408=798720,8880128=548864,9428992=253952} 2014-07-01 16:51:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: writing mira0435601-464 from 8081408 to 8880128 tid 1 2014-07-01 16:51:40,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: writing mira0435601-464 from 8880128 to 9428992 tid 2 2014-07-01 16:51:40,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: writing mira0435601-464 from 9428992 to 9682944 tid 3 2014-07-01 16:51:40,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2369: read oid 412 snap -1 2014-07-01 16:51:40,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: finishing write tid 1 to mira0435601-267 2014-07-01 16:51:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: finishing write tid 2 to mira0435601-267 2014-07-01 16:51:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: finishing write tid 4 to mira0435601-267 2014-07-01 16:51:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: finishing write tid 5 to mira0435601-267 2014-07-01 16:51:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 248 (ObjNum 1525 snap 0 seq_num 1525) dirty exists 2014-07-01 16:51:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 1 to mira0435601-466 2014-07-01 16:51:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: done (8 left) 2014-07-01 16:51:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 407 current snap is 0 2014-07-01 16:51:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: seq_num 1531 ranges {0=671744,671744=729088,1400832=73728} 2014-07-01 16:51:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: writing mira0435601-407 from 0 to 671744 tid 1 2014-07-01 16:51:40,787.787 INFO:teuthology.orchestra.run.mira043.stderr:testFileThenContainerDelete (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,787.787 INFO:teuthology.orchestra.run.mira043.stderr:testLastFileMarker (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: writing mira0435601-407 from 671744 to 1400832 tid 2 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: writing mira0435601-407 from 1400832 to 1474560 tid 3 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2371: delete oid 325 current snap is 0 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 2 to mira0435601-466 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 3 to mira0435601-466 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 4 to mira0435601-466 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 6 to mira0435601-466 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: finishing write tid 7 to mira0435601-466 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 200 (ObjNum 1526 snap 0 seq_num 1526) dirty exists 2014-07-01 16:51:40,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2363: expect (ObjNum 1271 snap 0 seq_num 1271) 2014-07-01 16:51:40,889.889 INFO:teuthology.orchestra.run.mira043.stderr:testMarkerLimitFileList (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:41,369.369 INFO:teuthology.orchestra.run.mira043.stderr:testPrefixAndLimit (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:41,371.371 INFO:teuthology.orchestra.run.mira043.stderr:testTooLongName (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:41,535.535 INFO:teuthology.orchestra.run.mira043.stderr:testUtf8Container (test.functional.tests.TestContainerUTF8) ... ok 2014-07-01 16:51:41,960.960 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:51:42,446.446 INFO:teuthology.orchestra.run.mira043.stderr:testBadHeaders (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:42,448.448 INFO:teuthology.orchestra.run.mira043.stderr:testBlankMetadataName (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:42,613.613 INFO:teuthology.orchestra.run.mira043.stderr:testChunkedPut (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:42,665.665 INFO:teuthology.orchestra.run.mira043.stderr:testContentTypeGuessing (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:42,669.669 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:50:44.721894', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882745560, u'health': u'HEALTH_OK', u'kb_used': 14035492, u'store_stats': {u'bytes_total': 6707415, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:50:54.539292', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886565492, u'health': u'HEALTH_OK', u'kb_used': 10215560, u'store_stats': {u'bytes_total': 8804044, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:51:11.322329', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886556160, u'health': u'HEALTH_OK', u'kb_used': 10224892, u'store_stats': {u'bytes_total': 9852680, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 36 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'num_pgs': 388, u'data_bytes': 1178772299, u'bytes_used': 3227025408, u'op_per_sec': 1038, u'version': 1688, u'pgs_by_state': [{u'count': 388, u'state_name': u'active+clean'}], u'read_bytes_sec': 6604894, u'write_bytes_sec': 2045839, u'bytes_avail': 4995355512832}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 627, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:51:42,669.669 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:51:42,669.669 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:51:42,669.669 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:51:42,670.670 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:51:42,670.670 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:51:42,670.670 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:51:42,836.836 INFO:teuthology.orchestra.run.mira043.stderr:testCopy (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:42,913.913 INFO:teuthology.orchestra.run.mira043.stderr:testCopy404s (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:43,005.005 INFO:teuthology.orchestra.run.mira043.stderr:testCopyDestinationSlashProblems (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:43,088.088 INFO:teuthology.orchestra.run.mira043.stderr:testCopyFromHeader (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:43,121.121 INFO:teuthology.orchestra.run.mira043.stderr:testCopyFromHeader404s (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:43,171.171 INFO:teuthology.orchestra.run.mira043.stderr:testCopyNoDestinationHeader (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:43,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 1 to mira0435601-445 2014-07-01 16:51:43,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 2 to mira0435601-445 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 3 to mira0435601-445 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 4 to mira0435601-445 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 6 to mira0435601-445 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 7 to mira0435601-445 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 209 (ObjNum 1527 snap 0 seq_num 1527) dirty exists 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 1 to mira0435601-494 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: done (9 left) 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2363: done (8 left) 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: done (7 left) 2014-07-01 16:51:43,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2371: done (6 left) 2014-07-01 16:51:43,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2372: read oid 352 snap -1 2014-07-01 16:51:43,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 2 to mira0435601-494 2014-07-01 16:51:43,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 3 to mira0435601-494 2014-07-01 16:51:43,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 4 to mira0435601-494 2014-07-01 16:51:43,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 5 to mira0435601-494 2014-07-01 16:51:44,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 7 to mira0435601-494 2014-07-01 16:51:44,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: finishing write tid 8 to mira0435601-494 2014-07-01 16:51:44,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 258 (ObjNum 1528 snap 0 seq_num 1528) dirty exists 2014-07-01 16:51:44,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 1 to mira0435601-70 2014-07-01 16:51:44,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 2 to mira0435601-70 2014-07-01 16:51:44,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 3 to mira0435601-70 2014-07-01 16:51:44,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 4 to mira0435601-70 2014-07-01 16:51:44,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 6 to mira0435601-70 2014-07-01 16:51:44,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: finishing write tid 7 to mira0435601-70 2014-07-01 16:51:44,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 70 v 250 (ObjNum 1529 snap 0 seq_num 1529) dirty exists 2014-07-01 16:51:44,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2367: expect (ObjNum 1377 snap 0 seq_num 1377) 2014-07-01 16:51:46,172.172 INFO:teuthology.orchestra.run.mira043.stderr:testDelete (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,187.187 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOfFileThatDoesNotExist (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,339.339 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteThen404s (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,339.339 INFO:teuthology.orchestra.run.mira043.stderr:testEtagResponse (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,339.339 INFO:teuthology.orchestra.run.mira043.stderr:testEtagWayoff (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,445.445 INFO:teuthology.orchestra.run.mira043.stderr:testFileCreate (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,460.460 INFO:teuthology.orchestra.run.mira043.stderr:testGetContentType (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,506.506 INFO:teuthology.orchestra.run.mira043.stderr:testGetOnFileThatDoesNotExist (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,506.506 INFO:teuthology.orchestra.run.mira043.stderr:testHead (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,506.506 INFO:teuthology.orchestra.run.mira043.stderr:testHeadOnFileThatDoesNotExist (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,732.732 INFO:teuthology.orchestra.run.mira043.stderr:testMetadataOnPost (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:46,961.961 INFO:teuthology.orchestra.run.mira043.stderr:testMetadataOnPut (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,032.032 INFO:teuthology.orchestra.run.mira043.stderr:testNameLimit (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,129.129 INFO:teuthology.orchestra.run.mira043.stderr:testNoContentLengthForPut (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,130.130 INFO:teuthology.orchestra.run.mira043.stderr:testPostOnFileThatDoesNotExist (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,130.130 INFO:teuthology.orchestra.run.mira043.stderr:testQuestionMarkInName (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,222.222 INFO:teuthology.orchestra.run.mira043.stderr:testRangedGets (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,369.369 INFO:teuthology.orchestra.run.mira043.stderr:testSerialization (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,623.623 INFO:teuthology.orchestra.run.mira043.stderr:testStackedOverwrite (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,626.626 INFO:teuthology.orchestra.run.mira043.stderr:testTooLongName (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:47,790.790 INFO:teuthology.orchestra.run.mira043.stderr:testZeroByteFile (test.functional.tests.TestFile) ... ok 2014-07-01 16:51:48,150.150 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:51:48,151.151 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:51:48,151.151 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 16:51:48,152.152 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:48,948.948 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:48,958.958 INFO:teuthology.orchestra.run.mira043.stderr:testIfMatch (test.functional.tests.TestFileComparison) ... ok 2014-07-01 16:51:49,172.172 INFO:teuthology.orchestra.run.mira043.stderr:testIfMatchAndUnmodified (test.functional.tests.TestFileComparison) ... ok 2014-07-01 16:51:49,332.332 INFO:teuthology.orchestra.run.mira043.stderr:testIfModifiedSince (test.functional.tests.TestFileComparison) ... ok 2014-07-01 16:51:49,451.451 INFO:teuthology.orchestra.run.mira043.stderr:testIfNoneMatch (test.functional.tests.TestFileComparison) ... ok 2014-07-01 16:51:49,599.599 INFO:teuthology.orchestra.run.mira043.stderr:testIfUnmodifiedSince (test.functional.tests.TestFileComparison) ... ok 2014-07-01 16:51:49,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: finishing write tid 1 to mira0435601-464 2014-07-01 16:51:49,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: finishing write tid 2 to mira0435601-464 2014-07-01 16:51:49,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: finishing write tid 3 to mira0435601-464 2014-07-01 16:51:49,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: finishing write tid 5 to mira0435601-464 2014-07-01 16:51:49,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: finishing write tid 6 to mira0435601-464 2014-07-01 16:51:49,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 343 (ObjNum 1530 snap 0 seq_num 1530) dirty exists 2014-07-01 16:51:49,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: finishing write tid 1 to mira0435601-407 2014-07-01 16:51:49,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: finishing write tid 2 to mira0435601-407 2014-07-01 16:51:49,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: finishing write tid 3 to mira0435601-407 2014-07-01 16:51:49,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: done (6 left) 2014-07-01 16:51:49,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: done (5 left) 2014-07-01 16:51:49,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2367: done (4 left) 2014-07-01 16:51:49,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: done (3 left) 2014-07-01 16:51:49,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2373: delete oid 202 current snap is 0 2014-07-01 16:51:49,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: finishing write tid 5 to mira0435601-407 2014-07-01 16:51:49,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2369: expect (ObjNum 1053 snap 0 seq_num 1053) 2014-07-01 16:51:50,375.375 INFO:teuthology.orchestra.run.mira043.stderr:testIfMatch (test.functional.tests.TestFileComparisonUTF8) ... ok 2014-07-01 16:51:50,586.586 INFO:teuthology.orchestra.run.mira043.stderr:testIfMatchAndUnmodified (test.functional.tests.TestFileComparisonUTF8) ... ok 2014-07-01 16:51:50,729.729 INFO:teuthology.orchestra.run.mira043.stderr:testIfModifiedSince (test.functional.tests.TestFileComparisonUTF8) ... ok 2014-07-01 16:51:50,892.892 INFO:teuthology.orchestra.run.mira043.stderr:testIfNoneMatch (test.functional.tests.TestFileComparisonUTF8) ... ok 2014-07-01 16:51:50,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: finishing write tid 6 to mira0435601-407 2014-07-01 16:51:50,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 407 v 258 (ObjNum 1531 snap 0 seq_num 1531) dirty exists 2014-07-01 16:51:50,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:2372: expect (ObjNum 351 snap 0 seq_num 351) 2014-07-01 16:51:51,089.089 INFO:teuthology.orchestra.run.mira043.stderr:testIfUnmodifiedSince (test.functional.tests.TestFileComparisonUTF8) ... ok 2014-07-01 16:51:51,630.630 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 16:51:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2369: done (3 left) 2014-07-01 16:51:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: done (2 left) 2014-07-01 16:51:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2372: done (1 left) 2014-07-01 16:51:51,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2373: done (0 left) 2014-07-01 16:51:51,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2374: read oid 181 snap -1 2014-07-01 16:51:51,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 16:51:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: seq_num 1534 ranges {5275648=491520,5767168=401408} 2014-07-01 16:51:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: writing mira0435601-313 from 5275648 to 5767168 tid 1 2014-07-01 16:51:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: writing mira0435601-313 from 5767168 to 6168576 tid 2 2014-07-01 16:51:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 16:51:51,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: seq_num 1535 ranges {1216512=516096,1732608=708608,2441216=569344,3010560=327680} 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: writing mira0435601-14 from 1216512 to 1732608 tid 1 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: writing mira0435601-14 from 1732608 to 2441216 tid 2 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: writing mira0435601-14 from 2441216 to 3010560 tid 3 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: writing mira0435601-14 from 3010560 to 3338240 tid 4 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 103 current snap is 0 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: seq_num 1536 ranges {0=790528,790528=696320,1486848=507904} 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: writing mira0435601-103 from 0 to 790528 tid 1 2014-07-01 16:51:51,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: writing mira0435601-103 from 790528 to 1486848 tid 2 2014-07-01 16:51:51,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: writing mira0435601-103 from 1486848 to 1994752 tid 3 2014-07-01 16:51:51,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 156 current snap is 0 2014-07-01 16:51:51,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: seq_num 1537 ranges {0=663552,663552=524288,1187840=671744,1859584=8192} 2014-07-01 16:51:51,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: writing mira0435601-156 from 0 to 663552 tid 1 2014-07-01 16:51:52,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: writing mira0435601-156 from 663552 to 1187840 tid 2 2014-07-01 16:51:52,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: writing mira0435601-156 from 1187840 to 1859584 tid 3 2014-07-01 16:51:52,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: writing mira0435601-156 from 1859584 to 1867776 tid 4 2014-07-01 16:51:52,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 438 current snap is 0 2014-07-01 16:51:52,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: seq_num 1538 ranges {4829184=737280,5566464=507904,6074368=294912} 2014-07-01 16:51:52,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: writing mira0435601-438 from 4829184 to 5566464 tid 1 2014-07-01 16:51:52,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: writing mira0435601-438 from 5566464 to 6074368 tid 2 2014-07-01 16:51:52,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: writing mira0435601-438 from 6074368 to 6369280 tid 3 2014-07-01 16:51:52,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: delete oid 446 current snap is 0 2014-07-01 16:51:52,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:2374: expect (ObjNum 1444 snap 0 seq_num 1444) 2014-07-01 16:51:52,654.654 INFO:teuthology.orchestra.run.mira043.stderr:testBadHeaders (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:52,658.658 INFO:teuthology.orchestra.run.mira043.stderr:testBlankMetadataName (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:52,852.852 INFO:teuthology.orchestra.run.mira043.stderr:testChunkedPut (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:52,920.920 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 16:51:52,938.938 INFO:teuthology.orchestra.run.mira043.stderr:testContentTypeGuessing (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,109.109 INFO:teuthology.orchestra.run.mira043.stderr:testCopy (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,189.189 INFO:teuthology.orchestra.run.mira043.stderr:testCopy404s (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,376.376 INFO:teuthology.orchestra.run.mira043.stderr:testCopyDestinationSlashProblems (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,396.396 INFO:teuthology.orchestra.run.mira043.stderr:testCopyFromHeader (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,417.417 INFO:teuthology.orchestra.run.mira043.stderr:testCopyFromHeader404s (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,582.582 INFO:teuthology.orchestra.run.mira043.stderr:testCopyNoDestinationHeader (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,583.583 INFO:teuthology.orchestra.run.mira043.stderr:testDelete (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,583.583 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteOfFileThatDoesNotExist (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,583.583 INFO:teuthology.orchestra.run.mira043.stderr:testDeleteThen404s (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,583.583 INFO:teuthology.orchestra.run.mira043.stderr:testEtagResponse (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,583.583 INFO:teuthology.orchestra.run.mira043.stderr:testEtagWayoff (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,791.791 INFO:teuthology.orchestra.run.mira043.stderr:testFileCreate (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,819.819 INFO:teuthology.orchestra.run.mira043.stderr:testGetContentType (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,822.822 INFO:teuthology.orchestra.run.mira043.stderr:testGetOnFileThatDoesNotExist (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:53,823.823 INFO:teuthology.orchestra.run.mira043.stderr:testHead (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,016.016 INFO:teuthology.orchestra.run.mira043.stderr:testHeadOnFileThatDoesNotExist (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,019.019 INFO:teuthology.orchestra.run.mira043.stderr:testMetadataOnPost (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,179.179 INFO:teuthology.orchestra.run.mira043.stderr:testMetadataOnPut (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,254.254 INFO:teuthology.orchestra.run.mira043.stderr:testNameLimit (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,378.378 INFO:teuthology.orchestra.run.mira043.stderr:testNoContentLengthForPut (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,378.378 INFO:teuthology.orchestra.run.mira043.stderr:testPostOnFileThatDoesNotExist (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,378.378 INFO:teuthology.orchestra.run.mira043.stderr:testQuestionMarkInName (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 1 to mira0435601-14 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 2 to mira0435601-14 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 3 to mira0435601-14 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: finishing write tid 1 to mira0435601-313 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: finishing write tid 2 to mira0435601-313 2014-07-01 16:51:54,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: finishing write tid 4 to mira0435601-313 2014-07-01 16:51:54,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 4 to mira0435601-14 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 6 to mira0435601-14 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: finishing write tid 1 to mira0435601-103 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: finishing write tid 2 to mira0435601-103 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: finishing write tid 3 to mira0435601-103 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: finishing write tid 5 to mira0435601-103 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: finishing write tid 1 to mira0435601-438 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: finishing write tid 2 to mira0435601-438 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: finishing write tid 3 to mira0435601-438 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: finishing write tid 5 to mira0435601-438 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: finishing write tid 5 to mira0435601-313 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 194 (ObjNum 1534 snap 0 seq_num 1534) dirty exists 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 1 to mira0435601-156 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 2 to mira0435601-156 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 3 to mira0435601-156 2014-07-01 16:51:54,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 4 to mira0435601-156 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 6 to mira0435601-156 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: finishing write tid 6 to mira0435601-103 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 211 (ObjNum 1536 snap 0 seq_num 1536) dirty exists 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: finishing write tid 7 to mira0435601-14 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 139 (ObjNum 1535 snap 0 seq_num 1535) dirty exists 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: finishing write tid 7 to mira0435601-156 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 141 (ObjNum 1537 snap 0 seq_num 1537) dirty exists 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: finishing write tid 6 to mira0435601-438 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 216 (ObjNum 1538 snap 0 seq_num 1538) dirty exists 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2374: done (6 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: done (5 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: done (4 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: done (3 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: done (2 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: done (1 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: done (0 left) 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2381: read oid 280 snap -1 2014-07-01 16:51:54,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: delete oid 139 current snap is 0 2014-07-01 16:51:54,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: done (1 left) 2014-07-01 16:51:54,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 281 current snap is 0 2014-07-01 16:51:54,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: seq_num 1541 ranges {4153344=106496} 2014-07-01 16:51:54,581.581 INFO:teuthology.orchestra.run.mira043.stderr:testRangedGets (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: writing mira0435601-281 from 4153344 to 4259840 tid 1 2014-07-01 16:51:54,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 359 current snap is 0 2014-07-01 16:51:54,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: seq_num 1542 ranges {6545408=430080,6975488=401408} 2014-07-01 16:51:54,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: writing mira0435601-359 from 6545408 to 6975488 tid 1 2014-07-01 16:51:54,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: writing mira0435601-359 from 6975488 to 7376896 tid 2 2014-07-01 16:51:54,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: read oid 300 snap -1 2014-07-01 16:51:54,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2381: expect (ObjNum 1503 snap 0 seq_num 1503) 2014-07-01 16:51:54,646.646 INFO:teuthology.orchestra.run.mira043.stderr:testSerialization (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,875.875 INFO:teuthology.orchestra.run.mira043.stderr:testStackedOverwrite (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:54,877.877 INFO:teuthology.orchestra.run.mira043.stderr:testTooLongName (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:55,078.078 INFO:teuthology.orchestra.run.mira043.stderr:testZeroByteFile (test.functional.tests.TestFileUTF8) ... ok 2014-07-01 16:51:55,079.079 INFO:teuthology.orchestra.run.mira043.stderr: 2014-07-01 16:51:55,079.079 INFO:teuthology.orchestra.run.mira043.stderr:---------------------------------------------------------------------- 2014-07-01 16:51:55,079.079 INFO:teuthology.orchestra.run.mira043.stderr:Ran 137 tests in 203.220s 2014-07-01 16:51:55,080.080 INFO:teuthology.orchestra.run.mira043.stderr: 2014-07-01 16:51:55,080.080 INFO:teuthology.orchestra.run.mira043.stderr:OK 2014-07-01 16:51:55,082.082 INFO:teuthology.orchestra.run.mira043:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user rm --uid foo.client.0 --purge-data' 2014-07-01 16:51:56,819.819 INFO:teuthology.orchestra.run.mira043:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user rm --uid bar.client.0 --purge-data' 2014-07-01 16:51:57,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: finishing write tid 1 to mira0435601-281 2014-07-01 16:51:57,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: finishing write tid 3 to mira0435601-281 2014-07-01 16:51:57,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: finishing write tid 4 to mira0435601-281 2014-07-01 16:51:57,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 146 (ObjNum 1541 snap 0 seq_num 1541) dirty exists 2014-07-01 16:51:57,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 1 to mira0435601-359 2014-07-01 16:51:57,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 2 to mira0435601-359 2014-07-01 16:51:57,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 4 to mira0435601-359 2014-07-01 16:51:57,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 5 to mira0435601-359 2014-07-01 16:51:57,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 229 (ObjNum 1542 snap 0 seq_num 1542) dirty exists 2014-07-01 16:51:57,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: expect (ObjNum 1036 snap 0 seq_num 1036) 2014-07-01 16:51:57,480.480 INFO:teuthology.task.swift:Removing swift... 2014-07-01 16:51:57,480.480 INFO:teuthology.orchestra.run.mira043:Running: 'rm -rf /home/ubuntu/cephtest/swift' 2014-07-01 16:51:57,928.928 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:51:57,929.929 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:51:57,929.929 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 16:51:57,929.929 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:51:58,150.150 INFO:teuthology.run_tasks:Running task rados... 2014-07-01 16:51:58,150.150 INFO:teuthology.task.rados:Beginning rados... 2014-07-01 16:51:58,150.150 DEBUG:teuthology.run_tasks:Unwinding manager rados 2014-07-01 16:51:58,150.150 INFO:teuthology.task.rados:joining rados 2014-07-01 16:51:58,150.150 INFO:teuthology.task.rados:clients are ['client.0'] 2014-07-01 16:51:58,150.150 INFO:teuthology.task.rados:starting run 0 out of 1 2014-07-01 16:51:58,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2381: done (3 left) 2014-07-01 16:51:58,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: done (2 left) 2014-07-01 16:51:58,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: done (1 left) 2014-07-01 16:51:58,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: done (0 left) 2014-07-01 16:51:58,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: read oid 319 snap -1 2014-07-01 16:51:58,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 16:51:58,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: seq_num 1543 ranges {2543616=458752,3002368=671744,3674112=462848,4136960=471040} 2014-07-01 16:51:58,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0435601-11 from 2543616 to 3002368 tid 1 2014-07-01 16:51:58,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0435601-11 from 3002368 to 3674112 tid 2 2014-07-01 16:51:58,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0435601-11 from 3674112 to 4136960 tid 3 2014-07-01 16:51:58,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0435601-11 from 4136960 to 4608000 tid 4 2014-07-01 16:51:58,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2388: delete oid 122 current snap is 0 2014-07-01 16:51:58,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: expect (ObjNum 1473 snap 0 seq_num 1473) 2014-07-01 16:51:58,713.713 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:51:59,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 1 to mira0435601-11 2014-07-01 16:51:59,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 2 to mira0435601-11 2014-07-01 16:51:59,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 3 to mira0435601-11 2014-07-01 16:51:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 4 to mira0435601-11 2014-07-01 16:51:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 6 to mira0435601-11 2014-07-01 16:51:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 7 to mira0435601-11 2014-07-01 16:51:59,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 129 (ObjNum 1543 snap 0 seq_num 1543) dirty exists 2014-07-01 16:51:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: done (2 left) 2014-07-01 16:51:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: done (1 left) 2014-07-01 16:51:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:2388: done (0 left) 2014-07-01 16:51:59,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: read oid 392 snap -1 2014-07-01 16:51:59,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 8 current snap is 0 2014-07-01 16:51:59,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: seq_num 1545 ranges {1662976=602112,2265088=483328,2748416=409600,3158016=311296} 2014-07-01 16:51:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: writing mira0435601-8 from 1662976 to 2265088 tid 1 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: writing mira0435601-8 from 2265088 to 2748416 tid 2 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: writing mira0435601-8 from 2748416 to 3158016 tid 3 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: writing mira0435601-8 from 3158016 to 3469312 tid 4 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 2 current snap is 0 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: seq_num 1546 ranges {0=630784} 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: writing mira0435601-2 from 0 to 630784 tid 1 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: delete oid 437 current snap is 0 2014-07-01 16:51:59,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: expect (ObjNum 1260 snap 0 seq_num 1260) 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 1 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 2 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 3 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 4 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 6 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: finishing write tid 7 to mira0435601-8 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 8 v 219 (ObjNum 1545 snap 0 seq_num 1545) dirty exists 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: finishing write tid 1 to mira0435601-2 2014-07-01 16:52:00,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: finishing write tid 3 to mira0435601-2 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: finishing write tid 4 to mira0435601-2 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 2 v 252 (ObjNum 1546 snap 0 seq_num 1546) dirty exists 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: done (3 left) 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: done (2 left) 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: done (1 left) 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: done (0 left) 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 55 current snap is 0 2014-07-01 16:52:00,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: seq_num 1548 ranges {2621440=421888,3043328=479232,3522560=323584} 2014-07-01 16:52:00,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: writing mira0435601-55 from 2621440 to 3043328 tid 1 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: writing mira0435601-55 from 3043328 to 3522560 tid 2 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: writing mira0435601-55 from 3522560 to 3846144 tid 3 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2394: read oid 465 snap -1 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: finishing write tid 1 to mira0435601-55 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 53 current snap is 0 2014-07-01 16:52:00,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: seq_num 1549 ranges {643072=49152} 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: writing mira0435601-53 from 643072 to 692224 tid 1 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 319 current snap is 0 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: seq_num 1550 ranges {2031616=720896,2752512=696320,3448832=528384,3977216=380928} 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: writing mira0435601-319 from 2031616 to 2752512 tid 1 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: writing mira0435601-319 from 2752512 to 3448832 tid 2 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: writing mira0435601-319 from 3448832 to 3977216 tid 3 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: writing mira0435601-319 from 3977216 to 4358144 tid 4 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 35 current snap is 0 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: seq_num 1551 ranges {3477504=643072} 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: writing mira0435601-35 from 3477504 to 4120576 tid 1 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: read oid 42 snap -1 2014-07-01 16:52:00,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 54 current snap is 0 2014-07-01 16:52:00,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: seq_num 1552 ranges {1511424=671744,2183168=708608,2891776=495616} 2014-07-01 16:52:00,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: writing mira0435601-54 from 1511424 to 2183168 tid 1 2014-07-01 16:52:00,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: writing mira0435601-54 from 2183168 to 2891776 tid 2 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: writing mira0435601-54 from 2891776 to 3387392 tid 3 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: seq_num 1553 ranges {0=696320} 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: writing mira0435601-228 from 0 to 696320 tid 1 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: read oid 485 snap -1 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: finishing write tid 2 to mira0435601-55 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: finishing write tid 3 to mira0435601-55 2014-07-01 16:52:00,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2394: done (8 left) 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2402: read oid 279 snap -1 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: read oid 154 snap -1 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: finishing write tid 5 to mira0435601-55 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 1 to mira0435601-53 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 3 to mira0435601-53 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: finishing write tid 6 to mira0435601-55 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 131 (ObjNum 1548 snap 0 seq_num 1548) dirty exists 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 1 to mira0435601-319 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 4 to mira0435601-53 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 196 (ObjNum 1549 snap 0 seq_num 1549) dirty exists 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 2 to mira0435601-319 2014-07-01 16:52:00,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 3 to mira0435601-319 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 4 to mira0435601-319 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 6 to mira0435601-319 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: finishing write tid 7 to mira0435601-319 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 170 (ObjNum 1550 snap 0 seq_num 1550) dirty exists 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 1 to mira0435601-35 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 3 to mira0435601-35 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 4 to mira0435601-35 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 260 (ObjNum 1551 snap 0 seq_num 1551) dirty exists 2014-07-01 16:52:00,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: expect (ObjNum 1468 snap 0 seq_num 1468) 2014-07-01 16:52:01,610.610 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:52:01,611.611 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 46' 2014-07-01 16:52:02,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: finishing write tid 1 to mira0435601-54 2014-07-01 16:52:02,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: finishing write tid 2 to mira0435601-54 2014-07-01 16:52:02,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: finishing write tid 3 to mira0435601-54 2014-07-01 16:52:02,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: finishing write tid 5 to mira0435601-54 2014-07-01 16:52:02,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: finishing write tid 6 to mira0435601-54 2014-07-01 16:52:02,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 174 (ObjNum 1552 snap 0 seq_num 1552) dirty exists 2014-07-01 16:52:02,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: finishing write tid 1 to mira0435601-228 2014-07-01 16:52:02,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: finishing write tid 3 to mira0435601-228 2014-07-01 16:52:02,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: expect (ObjNum 1491 snap 0 seq_num 1491) 2014-07-01 16:52:03,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: finishing write tid 4 to mira0435601-228 2014-07-01 16:52:03,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 143 (ObjNum 1553 snap 0 seq_num 1553) dirty exists 2014-07-01 16:52:03,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: expect (ObjNum 998 snap 0 seq_num 998) 2014-07-01 16:52:03,389.389 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 46 2014-07-01 16:52:03,403.403 INFO:teuthology.task.thrashosds.ceph_manager:creating pool_name unique_pool_2 2014-07-01 16:52:03,403.403 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool create unique_pool_2 16' 2014-07-01 16:52:04,433.433 INFO:teuthology.orchestra.run.mira079.stderr:pool 'unique_pool_2' created 2014-07-01 16:52:04,443.443 INFO:teuthology.orchestra.run.mira043:Running: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --op read 100 --op write 100 --op delete 50 --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_2' 2014-07-01 16:52:04,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight read -> 100 2014-07-01 16:52:04,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight write -> 100 2014-07-01 16:52:04,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight delete -> 50 2014-07-01 16:52:04,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight snap_create -> 50 2014-07-01 16:52:04,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight snap_remove -> 50 2014-07-01 16:52:04,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:adding op weight rollback -> 50 2014-07-01 16:52:04,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:ceph version 0.80.1-50-gd43e711 (d43e7113dd501aea1db33fdae30d56e96e9c3897) 2014-07-01 16:52:04,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:Configuration: 2014-07-01 16:52:04,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout: Number of operations: 4000 2014-07-01 16:52:04,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout: Number of objects: 500 2014-07-01 16:52:04,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout: Max in flight operations: 16 2014-07-01 16:52:04,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout: Object size (in bytes): 4000000 2014-07-01 16:52:04,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout: Write stride min: 400000 2014-07-01 16:52:04,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout: Write stride max: 800000 2014-07-01 16:52:04,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: write initial oid 1 2014-07-01 16:52:04,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: seq_num 0 ranges {612484=400000} 2014-07-01 16:52:04,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: writing mira0437250-1 from 612484 to 1012484 tid 1 2014-07-01 16:52:05,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: write initial oid 2 2014-07-01 16:52:05,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: seq_num 1 ranges {707712=1} 2014-07-01 16:52:05,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: writing mira0437250-2 from 707712 to 707713 tid 1 2014-07-01 16:52:05,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: write initial oid 3 2014-07-01 16:52:05,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: seq_num 2 ranges {403966=1} 2014-07-01 16:52:05,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: writing mira0437250-3 from 403966 to 403967 tid 1 2014-07-01 16:52:05,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: write initial oid 4 2014-07-01 16:52:05,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: seq_num 3 ranges {595325=774964,1898873=296452} 2014-07-01 16:52:05,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: writing mira0437250-4 from 595325 to 1370289 tid 1 2014-07-01 16:52:05,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: writing mira0437250-4 from 1898873 to 2195325 tid 2 2014-07-01 16:52:05,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: write initial oid 5 2014-07-01 16:52:05,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: seq_num 4 ranges {691579=630416,1891578=1} 2014-07-01 16:52:05,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: writing mira0437250-5 from 691579 to 1321995 tid 1 2014-07-01 16:52:05,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: writing mira0437250-5 from 1891578 to 1891579 tid 2 2014-07-01 16:52:05,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: write initial oid 6 2014-07-01 16:52:05,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: seq_num 5 ranges {504184=603244,1576736=527448} 2014-07-01 16:52:05,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: writing mira0437250-6 from 504184 to 1107428 tid 1 2014-07-01 16:52:05,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: writing mira0437250-6 from 1576736 to 2104184 tid 2 2014-07-01 16:52:05,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: write initial oid 7 2014-07-01 16:52:05,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: seq_num 6 ranges {696566=575049,1903350=770015,3073600=649760,3896565=1} 2014-07-01 16:52:05,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0437250-7 from 696566 to 1271615 tid 1 2014-07-01 16:52:05,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0437250-7 from 1903350 to 2673365 tid 2 2014-07-01 16:52:05,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0437250-7 from 3073600 to 3723360 tid 3 2014-07-01 16:52:05,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: writing mira0437250-7 from 3896565 to 3896566 tid 4 2014-07-01 16:52:05,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: write initial oid 8 2014-07-01 16:52:05,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: seq_num 7 ranges {792820=698277,2167882=688787,3273802=319018} 2014-07-01 16:52:05,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: writing mira0437250-8 from 792820 to 1491097 tid 1 2014-07-01 16:52:05,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: writing mira0437250-8 from 2167882 to 2856669 tid 2 2014-07-01 16:52:05,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: writing mira0437250-8 from 3273802 to 3592820 tid 3 2014-07-01 16:52:05,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: write initial oid 9 2014-07-01 16:52:05,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: seq_num 8 ranges {488050=553729,1480990=510406,2674003=614047} 2014-07-01 16:52:05,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0437250-9 from 488050 to 1041779 tid 1 2014-07-01 16:52:05,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0437250-9 from 1480990 to 1991396 tid 2 2014-07-01 16:52:05,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: writing mira0437250-9 from 2674003 to 3288050 tid 3 2014-07-01 16:52:05,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: write initial oid 10 2014-07-01 16:52:05,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: seq_num 9 ranges {32878=1} 2014-07-01 16:52:05,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: writing mira0437250-10 from 32878 to 32879 tid 1 2014-07-01 16:52:05,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: write initial oid 11 2014-07-01 16:52:05,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: seq_num 10 ranges {494061=649786,1294060=1} 2014-07-01 16:52:05,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0437250-11 from 494061 to 1143847 tid 1 2014-07-01 16:52:05,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: writing mira0437250-11 from 1294060 to 1294061 tid 2 2014-07-01 16:52:05,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: write initial oid 12 2014-07-01 16:52:05,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: seq_num 11 ranges {588267=400000} 2014-07-01 16:52:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: writing mira0437250-12 from 588267 to 988267 tid 1 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: write initial oid 13 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: seq_num 12 ranges {533097=628467,1733096=1} 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: writing mira0437250-13 from 533097 to 1161564 tid 1 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: writing mira0437250-13 from 1733096 to 1733097 tid 2 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: write initial oid 14 2014-07-01 16:52:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: seq_num 13 ranges {476902=752719,1677634=646854,2476901=1} 2014-07-01 16:52:05,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: writing mira0437250-14 from 476902 to 1229621 tid 1 2014-07-01 16:52:05,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: writing mira0437250-14 from 1677634 to 2324488 tid 2 2014-07-01 16:52:05,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: writing mira0437250-14 from 2476901 to 2476902 tid 3 2014-07-01 16:52:05,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: write initial oid 15 2014-07-01 16:52:05,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: seq_num 14 ranges {689508=724523,1908118=717049,2689507=1} 2014-07-01 16:52:05,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: writing mira0437250-15 from 689508 to 1414031 tid 1 2014-07-01 16:52:05,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: writing mira0437250-15 from 1908118 to 2625167 tid 2 2014-07-01 16:52:05,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: writing mira0437250-15 from 2689507 to 2689508 tid 3 2014-07-01 16:52:05,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: write initial oid 16 2014-07-01 16:52:05,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: seq_num 15 ranges {634338=447752,1869803=787244,3360533=73805} 2014-07-01 16:52:05,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: writing mira0437250-16 from 634338 to 1082090 tid 1 2014-07-01 16:52:05,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: writing mira0437250-16 from 1869803 to 2657047 tid 2 2014-07-01 16:52:05,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: writing mira0437250-16 from 3360533 to 3434338 tid 3 2014-07-01 16:52:05,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:05,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: done (9 left) 2014-07-01 16:52:05,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: done (8 left) 2014-07-01 16:52:05,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: done (7 left) 2014-07-01 16:52:05,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: done (6 left) 2014-07-01 16:52:05,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: done (5 left) 2014-07-01 16:52:05,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: done (4 left) 2014-07-01 16:52:05,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: done (3 left) 2014-07-01 16:52:05,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: done (2 left) 2014-07-01 16:52:05,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2402: done (1 left) 2014-07-01 16:52:05,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: done (0 left) 2014-07-01 16:52:05,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2404: read oid 204 snap -1 2014-07-01 16:52:05,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: seq_num 1554 ranges {0=413696,413696=520192,933888=577536,1511424=106496} 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: writing mira0435601-223 from 0 to 413696 tid 1 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: writing mira0435601-223 from 413696 to 933888 tid 2 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: writing mira0435601-223 from 933888 to 1511424 tid 3 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: writing mira0435601-223 from 1511424 to 1617920 tid 4 2014-07-01 16:52:05,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2404: expect (ObjNum 930 snap 0 seq_num 930) 2014-07-01 16:52:06,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 1 to mira0437250-6 2014-07-01 16:52:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 1 to mira0437250-5 2014-07-01 16:52:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 2 to mira0437250-5 2014-07-01 16:52:06,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 1 to mira0437250-4 2014-07-01 16:52:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 1 to mira0437250-14 2014-07-01 16:52:06,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 2 to mira0437250-4 2014-07-01 16:52:06,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 4 to mira0437250-4 2014-07-01 16:52:06,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 2 to mira0437250-6 2014-07-01 16:52:06,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 4 to mira0437250-6 2014-07-01 16:52:06,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 1 to mira0437250-7 2014-07-01 16:52:06,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: finishing write tid 5 to mira0437250-4 2014-07-01 16:52:06,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2014-07-01 16:52:06,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:4: done (15 left) 2014-07-01 16:52:06,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: write initial oid 17 2014-07-01 16:52:06,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: seq_num 16 ranges {178142=1} 2014-07-01 16:52:06,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: writing mira0437250-17 from 178142 to 178143 tid 1 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 2 to mira0437250-7 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 3 to mira0437250-7 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 4 to mira0437250-7 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 4 to mira0437250-5 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: finishing write tid 5 to mira0437250-5 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 3 (ObjNum 4 snap 0 seq_num 4) dirty exists 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:5: done (15 left) 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: write initial oid 18 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: seq_num 17 ranges {674397=677056,1947666=430482,2964006=619944,3874396=1} 2014-07-01 16:52:06,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0437250-18 from 674397 to 1351453 tid 1 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0437250-18 from 1947666 to 2378148 tid 2 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0437250-18 from 2964006 to 3583950 tid 3 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: writing mira0437250-18 from 3874396 to 3874397 tid 4 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 6 to mira0437250-7 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: finishing write tid 5 to mira0437250-6 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 3 (ObjNum 5 snap 0 seq_num 5) dirty exists 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 1 to mira0437250-1 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:6: done (15 left) 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: write initial oid 19 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: seq_num 18 ranges {618202=1} 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: writing mira0437250-19 from 618202 to 618203 tid 1 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 1 to mira0437250-13 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 1 to mira0437250-9 2014-07-01 16:52:06,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 2 to mira0437250-13 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 1 to mira0437250-3 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 4 to mira0437250-13 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 3 to mira0437250-3 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: finishing write tid 5 to mira0437250-13 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 6 (ObjNum 12 snap 0 seq_num 12) dirty exists 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: finishing write tid 4 to mira0437250-3 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 2 to mira0437250-14 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 3 to mira0437250-14 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 5 to mira0437250-14 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 1 to mira0437250-15 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:3: done (15 left) 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:13: done (14 left) 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: write initial oid 20 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: seq_num 19 ranges {678361=772089,1878360=1} 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0437250-20 from 678361 to 1450450 tid 1 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: writing mira0437250-20 from 1878360 to 1878361 tid 2 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: write initial oid 21 2014-07-01 16:52:06,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: seq_num 20 ranges {774614=628565,1574613=1} 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: writing mira0437250-21 from 774614 to 1403179 tid 1 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: writing mira0437250-21 from 1574613 to 1574614 tid 2 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 2 to mira0437250-9 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 3 to mira0437250-9 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 5 to mira0437250-9 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 1 to mira0437250-17 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: finishing write tid 7 to mira0437250-7 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 7 v 8 (ObjNum 6 snap 0 seq_num 6) dirty exists 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 3 to mira0437250-17 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 3 to mira0437250-1 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 1 to mira0437250-11 2014-07-01 16:52:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 2 to mira0437250-11 2014-07-01 16:52:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 4 to mira0437250-11 2014-07-01 16:52:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 2 to mira0437250-15 2014-07-01 16:52:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 3 to mira0437250-15 2014-07-01 16:52:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 5 to mira0437250-15 2014-07-01 16:52:06,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: finishing write tid 6 to mira0437250-15 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 4 (ObjNum 14 snap 0 seq_num 14) dirty exists 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: finishing write tid 6 to mira0437250-9 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 12 (ObjNum 8 snap 0 seq_num 8) dirty exists 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 1 to mira0437250-8 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:7: done (15 left) 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:9: done (14 left) 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:15: done (13 left) 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: write initial oid 22 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: seq_num 21 ranges {718420=751794,2180579=137841} 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: writing mira0437250-22 from 718420 to 1470214 tid 1 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: writing mira0437250-22 from 2180579 to 2318420 tid 2 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: write initial oid 23 2014-07-01 16:52:06,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: seq_num 22 ranges {414674=723598,1893687=120987} 2014-07-01 16:52:06,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: writing mira0437250-23 from 414674 to 1138272 tid 1 2014-07-01 16:52:06,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: writing mira0437250-23 from 1893687 to 2014674 tid 2 2014-07-01 16:52:06,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: write initial oid 24 2014-07-01 16:52:06,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: seq_num 23 ranges {474832=447851,1440525=638150,2784865=489967} 2014-07-01 16:52:06,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: writing mira0437250-24 from 474832 to 922683 tid 1 2014-07-01 16:52:06,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: writing mira0437250-24 from 1440525 to 2078675 tid 2 2014-07-01 16:52:06,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: writing mira0437250-24 from 2784865 to 3274832 tid 3 2014-07-01 16:52:06,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: finishing write tid 6 to mira0437250-14 2014-07-01 16:52:06,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 4 (ObjNum 13 snap 0 seq_num 13) dirty exists 2014-07-01 16:52:06,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: finishing write tid 4 to mira0437250-17 2014-07-01 16:52:06,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 17 v 2 (ObjNum 16 snap 0 seq_num 16) dirty exists 2014-07-01 16:52:06,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: finishing write tid 4 to mira0437250-1 2014-07-01 16:52:06,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2014-07-01 16:52:06,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 2 to mira0437250-8 2014-07-01 16:52:06,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 3 to mira0437250-8 2014-07-01 16:52:06,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 5 to mira0437250-8 2014-07-01 16:52:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: finishing write tid 6 to mira0437250-8 2014-07-01 16:52:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 8 v 4 (ObjNum 7 snap 0 seq_num 7) dirty exists 2014-07-01 16:52:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:1: done (15 left) 2014-07-01 16:52:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:8: done (14 left) 2014-07-01 16:52:06,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:14: done (13 left) 2014-07-01 16:52:06,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:17: done (12 left) 2014-07-01 16:52:06,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: write initial oid 25 2014-07-01 16:52:06,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: seq_num 24 ranges {19660=1} 2014-07-01 16:52:06,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: writing mira0437250-25 from 19660 to 19661 tid 1 2014-07-01 16:52:06,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: write initial oid 26 2014-07-01 16:52:06,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: seq_num 25 ranges {515915=426531,1401012=778540,2803186=433628,3715914=1} 2014-07-01 16:52:06,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0437250-26 from 515915 to 942446 tid 1 2014-07-01 16:52:06,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0437250-26 from 1401012 to 2179552 tid 2 2014-07-01 16:52:06,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0437250-26 from 2803186 to 3236814 tid 3 2014-07-01 16:52:06,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: writing mira0437250-26 from 3715914 to 3715915 tid 4 2014-07-01 16:52:06,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: write initial oid 27 2014-07-01 16:52:06,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: seq_num 26 ranges {459720=1} 2014-07-01 16:52:06,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: writing mira0437250-27 from 459720 to 459721 tid 1 2014-07-01 16:52:06,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: write initial oid 28 2014-07-01 16:52:06,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: seq_num 27 ranges {403527=771164,1203526=1} 2014-07-01 16:52:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: writing mira0437250-28 from 403527 to 1174691 tid 1 2014-07-01 16:52:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: writing mira0437250-28 from 1203526 to 1203527 tid 2 2014-07-01 16:52:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:06,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: finishing write tid 5 to mira0437250-11 2014-07-01 16:52:06,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 3 (ObjNum 10 snap 0 seq_num 10) dirty exists 2014-07-01 16:52:06,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 1 to mira0437250-18 2014-07-01 16:52:06,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 1 to mira0437250-12 2014-07-01 16:52:06,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 3 to mira0437250-12 2014-07-01 16:52:06,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: finishing write tid 4 to mira0437250-12 2014-07-01 16:52:06,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2014-07-01 16:52:06,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 2 to mira0437250-18 2014-07-01 16:52:06,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 1 to mira0437250-16 2014-07-01 16:52:06,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 1 to mira0437250-19 2014-07-01 16:52:06,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 2 to mira0437250-16 2014-07-01 16:52:06,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 3 to mira0437250-19 2014-07-01 16:52:06,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 1 to mira0437250-2 2014-07-01 16:52:06,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: finishing write tid 4 to mira0437250-19 2014-07-01 16:52:06,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 5 (ObjNum 18 snap 0 seq_num 18) dirty exists 2014-07-01 16:52:06,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 3 to mira0437250-2 2014-07-01 16:52:06,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 3 to mira0437250-16 2014-07-01 16:52:06,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 1 to mira0437250-10 2014-07-01 16:52:06,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 5 to mira0437250-16 2014-07-01 16:52:06,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 3 to mira0437250-10 2014-07-01 16:52:06,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: finishing write tid 6 to mira0437250-16 2014-07-01 16:52:06,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 6 (ObjNum 15 snap 0 seq_num 15) dirty exists 2014-07-01 16:52:06,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: finishing write tid 4 to mira0437250-10 2014-07-01 16:52:06,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 10 v 4 (ObjNum 9 snap 0 seq_num 9) dirty exists 2014-07-01 16:52:06,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: finishing write tid 4 to mira0437250-2 2014-07-01 16:52:06,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2014-07-01 16:52:06,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 3 to mira0437250-18 2014-07-01 16:52:06,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 4 to mira0437250-18 2014-07-01 16:52:06,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 6 to mira0437250-18 2014-07-01 16:52:06,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: finishing write tid 7 to mira0437250-18 2014-07-01 16:52:06,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 18 v 5 (ObjNum 17 snap 0 seq_num 17) dirty exists 2014-07-01 16:52:06,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 1 to mira0437250-20 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 2 to mira0437250-20 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 4 to mira0437250-20 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: finishing write tid 5 to mira0437250-20 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 6 (ObjNum 19 snap 0 seq_num 19) dirty exists 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 1 to mira0437250-21 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 2 to mira0437250-21 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 4 to mira0437250-21 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: finishing write tid 5 to mira0437250-21 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 3 (ObjNum 20 snap 0 seq_num 20) dirty exists 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 1 to mira0437250-22 2014-07-01 16:52:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 2 to mira0437250-22 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 4 to mira0437250-22 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: finishing write tid 5 to mira0437250-22 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 22 v 3 (ObjNum 21 snap 0 seq_num 21) dirty exists 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 1 to mira0437250-23 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 2 to mira0437250-23 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 4 to mira0437250-23 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: finishing write tid 5 to mira0437250-23 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 5 (ObjNum 22 snap 0 seq_num 22) dirty exists 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 1 to mira0437250-24 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 2 to mira0437250-24 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 1 to mira0437250-25 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 3 to mira0437250-25 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2: done (15 left) 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:10: done (14 left) 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:11: done (13 left) 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:12: done (12 left) 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:16: done (11 left) 2014-07-01 16:52:06,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:18: done (10 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:19: done (9 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:20: done (8 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:21: done (7 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:22: done (6 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:23: done (5 left) 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: write initial oid 29 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: seq_num 28 ranges {615108=626616,1415107=1} 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: writing mira0437250-29 from 615108 to 1241724 tid 1 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: writing mira0437250-29 from 1415107 to 1415108 tid 2 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: write initial oid 30 2014-07-01 16:52:06,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: seq_num 29 ranges {559938=749845,2033925=126013} 2014-07-01 16:52:06,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: writing mira0437250-30 from 559938 to 1309783 tid 1 2014-07-01 16:52:06,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: writing mira0437250-30 from 2033925 to 2159938 tid 2 2014-07-01 16:52:06,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: write initial oid 31 2014-07-01 16:52:06,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: seq_num 30 ranges {504768=722673,1845209=516012,2904767=1} 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: writing mira0437250-31 from 504768 to 1227441 tid 1 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: writing mira0437250-31 from 1845209 to 2361221 tid 2 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: writing mira0437250-31 from 2904767 to 2904768 tid 3 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: write initial oid 32 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: seq_num 31 ranges {599998=445901,1826093=737631,2599997=1} 2014-07-01 16:52:06,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0437250-32 from 599998 to 1045899 tid 1 2014-07-01 16:52:06,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0437250-32 from 1826093 to 2563724 tid 2 2014-07-01 16:52:06,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: writing mira0437250-32 from 2599997 to 2599998 tid 3 2014-07-01 16:52:06,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: write initial oid 33 2014-07-01 16:52:06,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: seq_num 32 ranges {412603=452778,1291649=654354,2555444=257159} 2014-07-01 16:52:06,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0437250-33 from 412603 to 865381 tid 1 2014-07-01 16:52:07,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0437250-33 from 1291649 to 1946003 tid 2 2014-07-01 16:52:07,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: writing mira0437250-33 from 2555444 to 2812603 tid 3 2014-07-01 16:52:07,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: write initial oid 34 2014-07-01 16:52:07,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: seq_num 33 ranges {604984=1} 2014-07-01 16:52:07,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: writing mira0437250-34 from 604984 to 604985 tid 1 2014-07-01 16:52:07,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: write initial oid 35 2014-07-01 16:52:07,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: seq_num 34 ranges {301238=1} 2014-07-01 16:52:07,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: writing mira0437250-35 from 301238 to 301239 tid 1 2014-07-01 16:52:07,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: write initial oid 36 2014-07-01 16:52:07,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: seq_num 35 ranges {796468=520639,1846525=616364,3007696=783773,3996467=1} 2014-07-01 16:52:07,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0437250-36 from 796468 to 1317107 tid 1 2014-07-01 16:52:07,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0437250-36 from 1846525 to 2462889 tid 2 2014-07-01 16:52:07,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0437250-36 from 3007696 to 3791469 tid 3 2014-07-01 16:52:07,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: writing mira0437250-36 from 3996467 to 3996468 tid 4 2014-07-01 16:52:07,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: write initial oid 37 2014-07-01 16:52:07,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: seq_num 36 ranges {588850=527515,1656785=132065} 2014-07-01 16:52:07,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: writing mira0437250-37 from 588850 to 1116365 tid 1 2014-07-01 16:52:07,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: writing mira0437250-37 from 1656785 to 1788850 tid 2 2014-07-01 16:52:07,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: write initial oid 38 2014-07-01 16:52:07,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: seq_num 37 ranges {401456=748919,1736869=264587} 2014-07-01 16:52:07,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: writing mira0437250-38 from 401456 to 1150375 tid 1 2014-07-01 16:52:07,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: writing mira0437250-38 from 1736869 to 2001456 tid 2 2014-07-01 16:52:07,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: write initial oid 39 2014-07-01 16:52:07,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: seq_num 38 ranges {746286=472148,1966330=462021,2746285=1} 2014-07-01 16:52:07,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: writing mira0437250-39 from 746286 to 1218434 tid 1 2014-07-01 16:52:07,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: writing mira0437250-39 from 1966330 to 2428351 tid 2 2014-07-01 16:52:07,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: writing mira0437250-39 from 2746285 to 2746286 tid 3 2014-07-01 16:52:07,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:07,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: finishing write tid 4 to mira0437250-25 2014-07-01 16:52:07,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 2 (ObjNum 24 snap 0 seq_num 24) dirty exists 2014-07-01 16:52:07,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 3 to mira0437250-24 2014-07-01 16:52:07,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 5 to mira0437250-24 2014-07-01 16:52:07,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: finishing write tid 6 to mira0437250-24 2014-07-01 16:52:07,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2014-07-01 16:52:07,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 1 to mira0437250-26 2014-07-01 16:52:07,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 2 to mira0437250-26 2014-07-01 16:52:07,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 1 to mira0437250-27 2014-07-01 16:52:07,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 3 to mira0437250-27 2014-07-01 16:52:07,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: finishing write tid 4 to mira0437250-27 2014-07-01 16:52:07,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 6 (ObjNum 26 snap 0 seq_num 26) dirty exists 2014-07-01 16:52:07,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 3 to mira0437250-26 2014-07-01 16:52:07,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 4 to mira0437250-26 2014-07-01 16:52:07,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 6 to mira0437250-26 2014-07-01 16:52:07,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: finishing write tid 7 to mira0437250-26 2014-07-01 16:52:07,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 7 (ObjNum 25 snap 0 seq_num 25) dirty exists 2014-07-01 16:52:07,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 1 to mira0437250-28 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 2 to mira0437250-28 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 4 to mira0437250-28 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: finishing write tid 5 to mira0437250-28 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 28 v 9 (ObjNum 27 snap 0 seq_num 27) dirty exists 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 1 to mira0437250-29 2014-07-01 16:52:07,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 2 to mira0437250-29 2014-07-01 16:52:07,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 4 to mira0437250-29 2014-07-01 16:52:07,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: finishing write tid 5 to mira0437250-29 2014-07-01 16:52:07,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 8 (ObjNum 28 snap 0 seq_num 28) dirty exists 2014-07-01 16:52:07,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 1 to mira0437250-30 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 2 to mira0437250-30 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 4 to mira0437250-30 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: finishing write tid 5 to mira0437250-30 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 7 (ObjNum 29 snap 0 seq_num 29) dirty exists 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 1 to mira0437250-31 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 2 to mira0437250-31 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 3 to mira0437250-31 2014-07-01 16:52:07,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 5 to mira0437250-31 2014-07-01 16:52:07,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: finishing write tid 6 to mira0437250-31 2014-07-01 16:52:07,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 16 (ObjNum 30 snap 0 seq_num 30) dirty exists 2014-07-01 16:52:07,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 1 to mira0437250-32 2014-07-01 16:52:07,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 2 to mira0437250-32 2014-07-01 16:52:07,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 3 to mira0437250-32 2014-07-01 16:52:07,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 5 to mira0437250-32 2014-07-01 16:52:07,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: finishing write tid 6 to mira0437250-32 2014-07-01 16:52:07,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 32 v 8 (ObjNum 31 snap 0 seq_num 31) dirty exists 2014-07-01 16:52:07,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 1 to mira0437250-33 2014-07-01 16:52:07,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 2 to mira0437250-33 2014-07-01 16:52:07,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 1 to mira0437250-34 2014-07-01 16:52:07,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 3 to mira0437250-34 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 3 to mira0437250-33 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: finishing write tid 4 to mira0437250-34 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 18 (ObjNum 33 snap 0 seq_num 33) dirty exists 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 5 to mira0437250-33 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 1 to mira0437250-35 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 3 to mira0437250-35 2014-07-01 16:52:07,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: finishing write tid 6 to mira0437250-33 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 11 (ObjNum 32 snap 0 seq_num 32) dirty exists 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: finishing write tid 4 to mira0437250-35 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 11 (ObjNum 34 snap 0 seq_num 34) dirty exists 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 1 to mira0437250-36 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 2 to mira0437250-36 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 3 to mira0437250-36 2014-07-01 16:52:07,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 4 to mira0437250-36 2014-07-01 16:52:07,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 6 to mira0437250-36 2014-07-01 16:52:07,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: finishing write tid 7 to mira0437250-36 2014-07-01 16:52:07,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 36 v 8 (ObjNum 35 snap 0 seq_num 35) dirty exists 2014-07-01 16:52:07,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 1 to mira0437250-37 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 2 to mira0437250-37 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 4 to mira0437250-37 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: finishing write tid 5 to mira0437250-37 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 1 to mira0437250-38 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 2 to mira0437250-38 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 4 to mira0437250-38 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: finishing write tid 5 to mira0437250-38 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 14 (ObjNum 37 snap 0 seq_num 37) dirty exists 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 1 to mira0437250-39 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:24: done (15 left) 2014-07-01 16:52:07,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:25: done (14 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:26: done (13 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:27: done (12 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:28: done (11 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:29: done (10 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:30: done (9 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:31: done (8 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:32: done (7 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:33: done (6 left) 2014-07-01 16:52:07,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:34: done (5 left) 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:35: done (4 left) 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:36: done (3 left) 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:37: done (2 left) 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:38: done (1 left) 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: write initial oid 40 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: seq_num 39 ranges {690091=443952,1625165=683641,2939125=550966} 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: writing mira0437250-40 from 690091 to 1134043 tid 1 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: writing mira0437250-40 from 1625165 to 2308806 tid 2 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: writing mira0437250-40 from 2939125 to 3490091 tid 3 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: write initial oid 41 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: seq_num 40 ranges {786345=450828,1774370=753836,3175423=10922} 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0437250-41 from 786345 to 1237173 tid 1 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0437250-41 from 1774370 to 2528206 tid 2 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: writing mira0437250-41 from 3175423 to 3186345 tid 3 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: write initial oid 42 2014-07-01 16:52:07,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: seq_num 41 ranges {446502=1} 2014-07-01 16:52:07,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: writing mira0437250-42 from 446502 to 446503 tid 1 2014-07-01 16:52:07,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: write initial oid 43 2014-07-01 16:52:07,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: seq_num 42 ranges {791333=400000} 2014-07-01 16:52:07,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: writing mira0437250-43 from 791333 to 1191333 tid 1 2014-07-01 16:52:07,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: write initial oid 44 2014-07-01 16:52:07,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: seq_num 43 ranges {486562=400000} 2014-07-01 16:52:07,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: writing mira0437250-44 from 486562 to 886562 tid 1 2014-07-01 16:52:07,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: write initial oid 45 2014-07-01 16:52:07,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: seq_num 44 ranges {430368=527614,1360755=269613} 2014-07-01 16:52:07,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: writing mira0437250-45 from 430368 to 957982 tid 1 2014-07-01 16:52:07,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: writing mira0437250-45 from 1360755 to 1630368 tid 2 2014-07-01 16:52:07,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: write initial oid 46 2014-07-01 16:52:07,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: seq_num 45 ranges {526622=499418,1326621=1} 2014-07-01 16:52:07,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0437250-46 from 526622 to 1026040 tid 1 2014-07-01 16:52:07,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: writing mira0437250-46 from 1326621 to 1326622 tid 2 2014-07-01 16:52:07,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: write initial oid 47 2014-07-01 16:52:07,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: seq_num 46 ranges {586780=622646,1668251=561503,2586779=1} 2014-07-01 16:52:07,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: writing mira0437250-47 from 586780 to 1209426 tid 1 2014-07-01 16:52:07,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: writing mira0437250-47 from 1668251 to 2229754 tid 2 2014-07-01 16:52:07,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: writing mira0437250-47 from 2586779 to 2586780 tid 3 2014-07-01 16:52:07,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: write initial oid 48 2014-07-01 16:52:07,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: seq_num 47 ranges {531609=594451,1630959=783122,3198528=133081} 2014-07-01 16:52:07,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: writing mira0437250-48 from 531609 to 1126060 tid 1 2014-07-01 16:52:07,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: writing mira0437250-48 from 1630959 to 2414081 tid 2 2014-07-01 16:52:07,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: writing mira0437250-48 from 3198528 to 3331609 tid 3 2014-07-01 16:52:07,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: write initial oid 49 2014-07-01 16:52:07,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: seq_num 48 ranges {627863=450927,1746115=453317,2849353=178510} 2014-07-01 16:52:07,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0437250-49 from 627863 to 1078790 tid 1 2014-07-01 16:52:07,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0437250-49 from 1746115 to 2199432 tid 2 2014-07-01 16:52:07,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: writing mira0437250-49 from 2849353 to 3027863 tid 3 2014-07-01 16:52:07,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: write initial oid 50 2014-07-01 16:52:07,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: seq_num 49 ranges {571669=422731,1556375=523512,2630355=660856,3771668=1} 2014-07-01 16:52:07,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0437250-50 from 571669 to 994400 tid 1 2014-07-01 16:52:07,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0437250-50 from 1556375 to 2079887 tid 2 2014-07-01 16:52:07,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0437250-50 from 2630355 to 3291211 tid 3 2014-07-01 16:52:07,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: writing mira0437250-50 from 3771668 to 3771669 tid 4 2014-07-01 16:52:07,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: write initial oid 51 2014-07-01 16:52:07,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: seq_num 50 ranges {632851=400000} 2014-07-01 16:52:07,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: writing mira0437250-51 from 632851 to 1032851 tid 1 2014-07-01 16:52:07,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: write initial oid 52 2014-07-01 16:52:07,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: seq_num 51 ranges {727055=1} 2014-07-01 16:52:07,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: writing mira0437250-52 from 727055 to 727056 tid 1 2014-07-01 16:52:07,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: write initial oid 53 2014-07-01 16:52:07,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: seq_num 52 ranges {671886=525664,1471885=1} 2014-07-01 16:52:07,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: writing mira0437250-53 from 671886 to 1197550 tid 1 2014-07-01 16:52:07,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: writing mira0437250-53 from 1471885 to 1471886 tid 2 2014-07-01 16:52:07,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: write initial oid 54 2014-07-01 16:52:07,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: seq_num 53 ranges {616716=497469,1541736=554693,2216715=1} 2014-07-01 16:52:07,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: writing mira0437250-54 from 616716 to 1114185 tid 1 2014-07-01 16:52:07,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: writing mira0437250-54 from 1541736 to 2096429 tid 2 2014-07-01 16:52:07,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: writing mira0437250-54 from 2216715 to 2216716 tid 3 2014-07-01 16:52:07,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:07,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 2 to mira0437250-39 2014-07-01 16:52:07,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 3 to mira0437250-39 2014-07-01 16:52:07,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 5 to mira0437250-39 2014-07-01 16:52:07,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: finishing write tid 6 to mira0437250-39 2014-07-01 16:52:07,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 39 v 18 (ObjNum 38 snap 0 seq_num 38) dirty exists 2014-07-01 16:52:07,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 1 to mira0437250-40 2014-07-01 16:52:07,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 2 to mira0437250-40 2014-07-01 16:52:07,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 3 to mira0437250-40 2014-07-01 16:52:07,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 5 to mira0437250-40 2014-07-01 16:52:07,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: finishing write tid 6 to mira0437250-40 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 10 (ObjNum 39 snap 0 seq_num 39) dirty exists 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 1 to mira0437250-41 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 1 to mira0437250-42 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 3 to mira0437250-42 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 2 to mira0437250-41 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 3 to mira0437250-41 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 5 to mira0437250-41 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: finishing write tid 4 to mira0437250-42 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: finishing write tid 6 to mira0437250-41 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 12 (ObjNum 40 snap 0 seq_num 40) dirty exists 2014-07-01 16:52:07,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 1 to mira0437250-43 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 3 to mira0437250-43 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: finishing write tid 4 to mira0437250-43 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 8 (ObjNum 42 snap 0 seq_num 42) dirty exists 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 1 to mira0437250-44 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 3 to mira0437250-44 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: finishing write tid 4 to mira0437250-44 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 44 v 10 (ObjNum 43 snap 0 seq_num 43) dirty exists 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 1 to mira0437250-45 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 2 to mira0437250-45 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 4 to mira0437250-45 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: finishing write tid 5 to mira0437250-45 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 5 (ObjNum 44 snap 0 seq_num 44) dirty exists 2014-07-01 16:52:07,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 1 to mira0437250-46 2014-07-01 16:52:07,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 2 to mira0437250-46 2014-07-01 16:52:07,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 4 to mira0437250-46 2014-07-01 16:52:07,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: finishing write tid 5 to mira0437250-46 2014-07-01 16:52:07,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 7 (ObjNum 45 snap 0 seq_num 45) dirty exists 2014-07-01 16:52:07,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 1 to mira0437250-47 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 2 to mira0437250-47 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 3 to mira0437250-47 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 5 to mira0437250-47 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: finishing write tid 6 to mira0437250-47 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 7 (ObjNum 46 snap 0 seq_num 46) dirty exists 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 1 to mira0437250-48 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 2 to mira0437250-48 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 3 to mira0437250-48 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 5 to mira0437250-48 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: finishing write tid 6 to mira0437250-48 2014-07-01 16:52:07,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 14 (ObjNum 47 snap 0 seq_num 47) dirty exists 2014-07-01 16:52:07,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 1 to mira0437250-49 2014-07-01 16:52:07,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 2 to mira0437250-49 2014-07-01 16:52:07,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 3 to mira0437250-49 2014-07-01 16:52:07,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 5 to mira0437250-49 2014-07-01 16:52:07,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: finishing write tid 6 to mira0437250-49 2014-07-01 16:52:07,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2014-07-01 16:52:07,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 1 to mira0437250-50 2014-07-01 16:52:07,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 2 to mira0437250-50 2014-07-01 16:52:07,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 3 to mira0437250-50 2014-07-01 16:52:07,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 4 to mira0437250-50 2014-07-01 16:52:07,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 6 to mira0437250-50 2014-07-01 16:52:07,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: finishing write tid 7 to mira0437250-50 2014-07-01 16:52:07,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 15 (ObjNum 49 snap 0 seq_num 49) dirty exists 2014-07-01 16:52:07,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 1 to mira0437250-52 2014-07-01 16:52:07,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 1 to mira0437250-51 2014-07-01 16:52:07,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 3 to mira0437250-52 2014-07-01 16:52:07,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 3 to mira0437250-51 2014-07-01 16:52:07,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: finishing write tid 4 to mira0437250-52 2014-07-01 16:52:07,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 52 v 8 (ObjNum 51 snap 0 seq_num 51) dirty exists 2014-07-01 16:52:07,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: finishing write tid 4 to mira0437250-51 2014-07-01 16:52:07,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 51 v 17 (ObjNum 50 snap 0 seq_num 50) dirty exists 2014-07-01 16:52:07,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 1 to mira0437250-53 2014-07-01 16:52:07,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:39: done (15 left) 2014-07-01 16:52:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:40: done (14 left) 2014-07-01 16:52:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:41: done (13 left) 2014-07-01 16:52:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:42: done (12 left) 2014-07-01 16:52:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:43: done (11 left) 2014-07-01 16:52:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:44: done (10 left) 2014-07-01 16:52:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:45: done (9 left) 2014-07-01 16:52:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:46: done (8 left) 2014-07-01 16:52:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:47: done (7 left) 2014-07-01 16:52:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:48: done (6 left) 2014-07-01 16:52:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:49: done (5 left) 2014-07-01 16:52:07,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:50: done (4 left) 2014-07-01 16:52:07,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:51: done (3 left) 2014-07-01 16:52:07,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:52: done (2 left) 2014-07-01 16:52:07,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: write initial oid 55 2014-07-01 16:52:07,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: seq_num 54 ranges {711945=620697,1805243=106702} 2014-07-01 16:52:07,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: writing mira0437250-55 from 711945 to 1332642 tid 1 2014-07-01 16:52:07,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: writing mira0437250-55 from 1805243 to 1911945 tid 2 2014-07-01 16:52:07,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: write initial oid 56 2014-07-01 16:52:07,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: seq_num 55 ranges {773127=593526,2133904=481580,3173126=1} 2014-07-01 16:52:07,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: writing mira0437250-56 from 773127 to 1366653 tid 1 2014-07-01 16:52:07,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: writing mira0437250-56 from 2133904 to 2615484 tid 2 2014-07-01 16:52:07,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: writing mira0437250-56 from 3173126 to 3173127 tid 3 2014-07-01 16:52:07,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: write initial oid 57 2014-07-01 16:52:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: seq_num 56 ranges {716933=600402,1847012=400351,2935061=740198,3916932=1} 2014-07-01 16:52:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0437250-57 from 716933 to 1317335 tid 1 2014-07-01 16:52:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0437250-57 from 1847012 to 2247363 tid 2 2014-07-01 16:52:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0437250-57 from 2935061 to 3675259 tid 3 2014-07-01 16:52:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: writing mira0437250-57 from 3916932 to 3916933 tid 4 2014-07-01 16:52:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: write initial oid 58 2014-07-01 16:52:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: seq_num 57 ranges {413187=572206,1561145=621970,2736287=474541,3613186=1} 2014-07-01 16:52:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0437250-58 from 413187 to 985393 tid 1 2014-07-01 16:52:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0437250-58 from 1561145 to 2183115 tid 2 2014-07-01 16:52:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0437250-58 from 2736287 to 3210828 tid 3 2014-07-01 16:52:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: writing mira0437250-58 from 3613186 to 3613187 tid 4 2014-07-01 16:52:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: write initial oid 59 2014-07-01 16:52:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: seq_num 58 ranges {509440=695435,1791629=443589,2688936=620504} 2014-07-01 16:52:07,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0437250-59 from 509440 to 1204875 tid 1 2014-07-01 16:52:07,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0437250-59 from 1791629 to 2235218 tid 2 2014-07-01 16:52:07,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: writing mira0437250-59 from 2688936 to 3309440 tid 3 2014-07-01 16:52:07,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: write initial oid 60 2014-07-01 16:52:07,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: seq_num 59 ranges {418174=516839,1567841=50333} 2014-07-01 16:52:07,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: writing mira0437250-60 from 418174 to 935013 tid 1 2014-07-01 16:52:08,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: writing mira0437250-60 from 1567841 to 1618174 tid 2 2014-07-01 16:52:08,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: write initial oid 61 2014-07-01 16:52:08,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: seq_num 60 ranges {513404=523715,1313403=1} 2014-07-01 16:52:08,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: writing mira0437250-61 from 513404 to 1037119 tid 1 2014-07-01 16:52:08,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: writing mira0437250-61 from 1313403 to 1313404 tid 2 2014-07-01 16:52:08,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: write initial oid 62 2014-07-01 16:52:08,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: seq_num 61 ranges {608634=400000} 2014-07-01 16:52:08,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: writing mira0437250-62 from 608634 to 1008634 tid 1 2014-07-01 16:52:08,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: write initial oid 63 2014-07-01 16:52:08,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: seq_num 62 ranges {402039=619772,1756765=475794,2802038=1} 2014-07-01 16:52:08,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: writing mira0437250-63 from 402039 to 1021811 tid 1 2014-07-01 16:52:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: writing mira0437250-63 from 1756765 to 2232559 tid 2 2014-07-01 16:52:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: writing mira0437250-63 from 2802038 to 2802039 tid 3 2014-07-01 16:52:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: write initial oid 64 2014-07-01 16:52:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: seq_num 63 ranges {498293=591576,1587249=545989,2498292=1} 2014-07-01 16:52:08,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0437250-64 from 498293 to 1089869 tid 1 2014-07-01 16:52:08,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0437250-64 from 1587249 to 2133238 tid 2 2014-07-01 16:52:08,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: writing mira0437250-64 from 2498292 to 2498293 tid 3 2014-07-01 16:52:08,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: write initial oid 65 2014-07-01 16:52:08,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: seq_num 64 ranges {709875=714805,2216710=493165} 2014-07-01 16:52:08,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: writing mira0437250-65 from 709875 to 1424680 tid 1 2014-07-01 16:52:08,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: writing mira0437250-65 from 2216710 to 2709875 tid 2 2014-07-01 16:52:08,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: write initial oid 66 2014-07-01 16:52:08,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: seq_num 65 ranges {503280=1} 2014-07-01 16:52:08,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: writing mira0437250-66 from 503280 to 503281 tid 1 2014-07-01 16:52:08,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: write initial oid 67 2014-07-01 16:52:08,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: seq_num 66 ranges {198509=1} 2014-07-01 16:52:08,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: writing mira0437250-67 from 198509 to 198510 tid 1 2014-07-01 16:52:08,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: write initial oid 68 2014-07-01 16:52:08,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: seq_num 67 ranges {694764=666314,1856258=461842,2826493=425710,3795170=99594} 2014-07-01 16:52:08,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0437250-68 from 694764 to 1361078 tid 1 2014-07-01 16:52:08,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 1 to mira0435601-223 2014-07-01 16:52:08,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 2 to mira0435601-223 2014-07-01 16:52:08,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: read oid 299 snap -1 2014-07-01 16:52:08,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2404: done (2 left) 2014-07-01 16:52:08,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2407: read oid 304 snap -1 2014-07-01 16:52:08,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 3 to mira0435601-223 2014-07-01 16:52:08,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 4 to mira0435601-223 2014-07-01 16:52:08,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 6 to mira0435601-223 2014-07-01 16:52:08,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 182 current snap is 0 2014-07-01 16:52:08,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: seq_num 1555 ranges {0=442368} 2014-07-01 16:52:08,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0437250-68 from 1856258 to 2318100 tid 2 2014-07-01 16:52:08,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: writing mira0435601-182 from 0 to 442368 tid 1 2014-07-01 16:52:08,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: read oid 191 snap -1 2014-07-01 16:52:08,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: finishing write tid 7 to mira0435601-223 2014-07-01 16:52:08,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 207 (ObjNum 1554 snap 0 seq_num 1554) dirty exists 2014-07-01 16:52:08,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: done (4 left) 2014-07-01 16:52:08,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: read oid 313 snap -1 2014-07-01 16:52:08,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: read oid 118 snap -1 2014-07-01 16:52:08,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 423 current snap is 0 2014-07-01 16:52:08,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: seq_num 1556 ranges {6070272=114688} 2014-07-01 16:52:08,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0437250-68 from 2826493 to 3252203 tid 3 2014-07-01 16:52:08,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: writing mira0437250-68 from 3795170 to 3894764 tid 4 2014-07-01 16:52:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 2 to mira0437250-53 2014-07-01 16:52:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 4 to mira0437250-53 2014-07-01 16:52:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: finishing write tid 5 to mira0437250-53 2014-07-01 16:52:08,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 11 (ObjNum 52 snap 0 seq_num 52) dirty exists 2014-07-01 16:52:08,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 1 to mira0437250-54 2014-07-01 16:52:08,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 2 to mira0437250-54 2014-07-01 16:52:08,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 3 to mira0437250-54 2014-07-01 16:52:08,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 5 to mira0437250-54 2014-07-01 16:52:08,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: finishing write tid 6 to mira0437250-54 2014-07-01 16:52:08,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 11 (ObjNum 53 snap 0 seq_num 53) dirty exists 2014-07-01 16:52:08,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 1 to mira0437250-55 2014-07-01 16:52:08,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 2 to mira0437250-55 2014-07-01 16:52:08,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 4 to mira0437250-55 2014-07-01 16:52:08,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: finishing write tid 5 to mira0437250-55 2014-07-01 16:52:08,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 20 (ObjNum 54 snap 0 seq_num 54) dirty exists 2014-07-01 16:52:08,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 1 to mira0437250-57 2014-07-01 16:52:08,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 1 to mira0437250-56 2014-07-01 16:52:08,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 2 to mira0437250-56 2014-07-01 16:52:08,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 1 to mira0437250-60 2014-07-01 16:52:08,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 2 to mira0437250-60 2014-07-01 16:52:08,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 4 to mira0437250-60 2014-07-01 16:52:08,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 2 to mira0437250-57 2014-07-01 16:52:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 1 to mira0437250-64 2014-07-01 16:52:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 2 to mira0437250-64 2014-07-01 16:52:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 3 to mira0437250-64 2014-07-01 16:52:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:53: done (15 left) 2014-07-01 16:52:08,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:54: done (14 left) 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:55: done (13 left) 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: write initial oid 69 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: seq_num 68 ranges {603498=789542,2050646=152852} 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: writing mira0437250-69 from 603498 to 1393040 tid 1 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: writing mira0437250-69 from 2050646 to 2203498 tid 2 2014-07-01 16:52:08,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: write initial oid 70 2014-07-01 16:52:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: seq_num 69 ranges {698728=646018,1895978=2750} 2014-07-01 16:52:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: writing mira0437250-70 from 698728 to 1344746 tid 1 2014-07-01 16:52:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: writing mira0437250-70 from 1895978 to 1898728 tid 2 2014-07-01 16:52:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: writing mira0435601-423 from 6070272 to 6184960 tid 1 2014-07-01 16:52:08,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: read oid 411 snap -1 2014-07-01 16:52:08,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2407: expect (ObjNum 1300 snap 0 seq_num 1300) 2014-07-01 16:52:08,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: write initial oid 71 2014-07-01 16:52:08,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: seq_num 70 ranges {794981=617823,1594980=1} 2014-07-01 16:52:08,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0437250-71 from 794981 to 1412804 tid 1 2014-07-01 16:52:08,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: writing mira0437250-71 from 1594980 to 1594981 tid 2 2014-07-01 16:52:08,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 5 to mira0437250-64 2014-07-01 16:52:08,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 1 to mira0437250-61 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 3 to mira0437250-57 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 2 to mira0437250-61 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 4 to mira0437250-57 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 4 to mira0437250-61 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 6 to mira0437250-57 2014-07-01 16:52:08,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: finishing write tid 7 to mira0437250-57 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 14 (ObjNum 56 snap 0 seq_num 56) dirty exists 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 1 to mira0437250-63 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 2 to mira0437250-63 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 3 to mira0437250-63 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 3 to mira0437250-56 2014-07-01 16:52:08,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 5 to mira0437250-56 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 5 to mira0437250-63 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 1 to mira0437250-58 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 2 to mira0437250-58 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 3 to mira0437250-58 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 4 to mira0437250-58 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 6 to mira0437250-58 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:57: done (15 left) 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: write initial oid 72 2014-07-01 16:52:08,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: seq_num 71 ranges {738787=589627,1936722=402065} 2014-07-01 16:52:08,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: writing mira0437250-72 from 738787 to 1328414 tid 1 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: writing mira0437250-72 from 1936722 to 2338787 tid 2 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: finishing write tid 6 to mira0437250-63 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 18 (ObjNum 62 snap 0 seq_num 62) dirty exists 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 1 to mira0437250-62 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:63: done (15 left) 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: write initial oid 73 2014-07-01 16:52:08,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: seq_num 72 ranges {683617=712855,2050854=714642,3083616=1} 2014-07-01 16:52:08,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0437250-73 from 683617 to 1396472 tid 1 2014-07-01 16:52:08,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2407: done (7 left) 2014-07-01 16:52:08,404.404 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:08,404.404 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:08,405.405 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:52:08,405.405 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:52:08,405.405 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:52:08,406.406 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 109 current snap is 0 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: seq_num 1557 ranges {3895296=634880,4530176=401408} 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0437250-73 from 2050854 to 2765496 tid 2 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: writing mira0437250-73 from 3083616 to 3083617 tid 3 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 3 to mira0437250-62 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 1 to mira0437250-59 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 2 to mira0437250-59 2014-07-01 16:52:08,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 3 to mira0437250-59 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: finishing write tid 5 to mira0437250-61 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 15 (ObjNum 60 snap 0 seq_num 60) dirty exists 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 5 to mira0437250-59 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 1 to mira0437250-65 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: finishing write tid 6 to mira0437250-56 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 22 (ObjNum 55 snap 0 seq_num 55) dirty exists 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 1 to mira0437250-66 2014-07-01 16:52:08,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: finishing write tid 6 to mira0437250-59 2014-07-01 16:52:08,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 15 (ObjNum 58 snap 0 seq_num 58) dirty exists 2014-07-01 16:52:08,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:56: done (15 left) 2014-07-01 16:52:08,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:59: done (14 left) 2014-07-01 16:52:08,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:61: done (13 left) 2014-07-01 16:52:08,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: write initial oid 74 2014-07-01 16:52:08,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: seq_num 73 ranges {496222=570356,1731962=668485,2877748=418474} 2014-07-01 16:52:08,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: writing mira0437250-74 from 496222 to 1066578 tid 1 2014-07-01 16:52:08,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: writing mira0437250-74 from 1731962 to 2400447 tid 2 2014-07-01 16:52:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: writing mira0437250-74 from 2877748 to 3296222 tid 3 2014-07-01 16:52:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: write initial oid 75 2014-07-01 16:52:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: seq_num 74 ranges {40027=1} 2014-07-01 16:52:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: writing mira0437250-75 from 40027 to 40028 tid 1 2014-07-01 16:52:08,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: write initial oid 76 2014-07-01 16:52:08,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: seq_num 75 ranges {784857=1} 2014-07-01 16:52:08,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: writing mira0437250-76 from 784857 to 784858 tid 1 2014-07-01 16:52:08,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 1 to mira0437250-67 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 3 to mira0437250-67 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: finishing write tid 5 to mira0437250-60 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 60 v 14 (ObjNum 59 snap 0 seq_num 59) dirty exists 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: finishing write tid 7 to mira0437250-58 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 13 (ObjNum 57 snap 0 seq_num 57) dirty exists 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 2 to mira0437250-65 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: writing mira0435601-109 from 3895296 to 4530176 tid 1 2014-07-01 16:52:08,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: writing mira0435601-109 from 4530176 to 4931584 tid 2 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2415: read oid 335 snap -1 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 1 to mira0435601-182 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 3 to mira0435601-182 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2416: read oid 158 snap -1 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2417: read oid 362 snap -1 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 456 current snap is 0 2014-07-01 16:52:08,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: seq_num 1558 ranges {2904064=659456,3563520=503808,4067328=577536,4644864=524288} 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 4 to mira0437250-65 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 3 to mira0437250-66 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: finishing write tid 4 to mira0437250-66 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 20 (ObjNum 65 snap 0 seq_num 65) dirty exists 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: finishing write tid 4 to mira0437250-67 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 15 (ObjNum 66 snap 0 seq_num 66) dirty exists 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: finishing write tid 4 to mira0437250-62 2014-07-01 16:52:08,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 10 (ObjNum 61 snap 0 seq_num 61) dirty exists 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: finishing write tid 6 to mira0437250-64 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 14 (ObjNum 63 snap 0 seq_num 63) dirty exists 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: finishing write tid 5 to mira0437250-65 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 18 (ObjNum 64 snap 0 seq_num 64) dirty exists 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 1 to mira0437250-68 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:58: done (15 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:60: done (14 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:62: done (13 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:64: done (12 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:65: done (11 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:66: done (10 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:67: done (9 left) 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: write initial oid 77 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: seq_num 76 ranges {479063=1} 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: writing mira0437250-77 from 479063 to 479064 tid 1 2014-07-01 16:52:08,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: write initial oid 78 2014-07-01 16:52:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: seq_num 77 ranges {540245=645093,1598923=141322} 2014-07-01 16:52:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: writing mira0437250-78 from 540245 to 1185338 tid 1 2014-07-01 16:52:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: writing mira0437250-78 from 1598923 to 1740245 tid 2 2014-07-01 16:52:08,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: write initial oid 79 2014-07-01 16:52:08,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: seq_num 78 ranges {485075=768321,1829407=522309,2485074=1} 2014-07-01 16:52:08,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: writing mira0435601-456 from 2904064 to 3563520 tid 1 2014-07-01 16:52:08,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: writing mira0435601-456 from 3563520 to 4067328 tid 2 2014-07-01 16:52:08,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: writing mira0435601-456 from 4067328 to 4644864 tid 3 2014-07-01 16:52:08,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: writing mira0437250-79 from 485075 to 1253396 tid 1 2014-07-01 16:52:08,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: writing mira0435601-456 from 4644864 to 5169152 tid 4 2014-07-01 16:52:08,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 442 current snap is 0 2014-07-01 16:52:08,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: seq_num 1559 ranges {3043328=688128,3731456=401408} 2014-07-01 16:52:08,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: writing mira0437250-79 from 1829407 to 2351716 tid 2 2014-07-01 16:52:08,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: writing mira0437250-79 from 2485074 to 2485075 tid 3 2014-07-01 16:52:08,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: write initial oid 80 2014-07-01 16:52:08,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: seq_num 79 ranges {580305=740126,1791092=389213} 2014-07-01 16:52:08,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: writing mira0437250-80 from 580305 to 1320431 tid 1 2014-07-01 16:52:08,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: writing mira0437250-80 from 1791092 to 2180305 tid 2 2014-07-01 16:52:08,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: write initial oid 81 2014-07-01 16:52:08,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: seq_num 80 ranges {525135=464378,1506248=662699,2783478=141657} 2014-07-01 16:52:08,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0437250-81 from 525135 to 989513 tid 1 2014-07-01 16:52:08,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0437250-81 from 1506248 to 2168947 tid 2 2014-07-01 16:52:08,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: writing mira0437250-81 from 2783478 to 2925135 tid 3 2014-07-01 16:52:08,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: write initial oid 82 2014-07-01 16:52:08,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: seq_num 81 ranges {468940=568406,1716507=484319,2680831=550345,3668939=1} 2014-07-01 16:52:08,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0437250-82 from 468940 to 1037346 tid 1 2014-07-01 16:52:08,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: writing mira0435601-442 from 3043328 to 3731456 tid 1 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: writing mira0435601-442 from 3731456 to 4132864 tid 2 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 322 current snap is 0 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: seq_num 1560 ranges {0=454656} 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: writing mira0435601-322 from 0 to 454656 tid 1 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: delete oid 444 current snap is 0 2014-07-01 16:52:08,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 4 to mira0435601-182 2014-07-01 16:52:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 261 (ObjNum 1555 snap 0 seq_num 1555) dirty exists 2014-07-01 16:52:08,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:2417: expect (ObjNum 1352 snap 0 seq_num 1352) 2014-07-01 16:52:08,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0437250-82 from 1716507 to 2200826 tid 2 2014-07-01 16:52:08,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0437250-82 from 2680831 to 3231176 tid 3 2014-07-01 16:52:08,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: writing mira0437250-82 from 3668939 to 3668940 tid 4 2014-07-01 16:52:08,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: write initial oid 83 2014-07-01 16:52:08,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: seq_num 82 ranges {530122=400000} 2014-07-01 16:52:08,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: writing mira0437250-83 from 530122 to 930122 tid 1 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 2 to mira0437250-68 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 3 to mira0437250-68 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 4 to mira0437250-68 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 6 to mira0437250-68 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: finishing write tid 7 to mira0437250-68 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 20 (ObjNum 67 snap 0 seq_num 67) dirty exists 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 1 to mira0437250-71 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 2 to mira0437250-71 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 4 to mira0437250-71 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: finishing write tid 5 to mira0437250-71 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 9 (ObjNum 70 snap 0 seq_num 70) dirty exists 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 1 to mira0437250-72 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 2 to mira0437250-72 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:68: done (15 left) 2014-07-01 16:52:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:71: done (14 left) 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: write initial oid 84 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: seq_num 83 ranges {626375=1} 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: writing mira0437250-84 from 626375 to 626376 tid 1 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: write initial oid 85 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: seq_num 84 ranges {320581=1} 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: writing mira0437250-85 from 320581 to 320582 tid 1 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 4 to mira0437250-72 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: finishing write tid 5 to mira0437250-72 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 23 (ObjNum 71 snap 0 seq_num 71) dirty exists 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 1 to mira0437250-73 2014-07-01 16:52:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 2 to mira0437250-73 2014-07-01 16:52:08,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 3 to mira0437250-73 2014-07-01 16:52:08,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:72: done (15 left) 2014-07-01 16:52:08,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: write initial oid 86 2014-07-01 16:52:08,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: seq_num 85 ranges {513987=643144,1584492=529495} 2014-07-01 16:52:08,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: writing mira0437250-86 from 513987 to 1157131 tid 1 2014-07-01 16:52:08,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: writing mira0437250-86 from 1584492 to 2113987 tid 2 2014-07-01 16:52:08,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:08,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 5 to mira0437250-73 2014-07-01 16:52:08,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: finishing write tid 6 to mira0437250-73 2014-07-01 16:52:08,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 11 (ObjNum 72 snap 0 seq_num 72) dirty exists 2014-07-01 16:52:08,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 1 to mira0437250-74 2014-07-01 16:52:08,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 2 to mira0437250-74 2014-07-01 16:52:08,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 1 to mira0437250-75 2014-07-01 16:52:08,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 3 to mira0437250-75 2014-07-01 16:52:08,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: finishing write tid 4 to mira0437250-75 2014-07-01 16:52:08,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 11 (ObjNum 74 snap 0 seq_num 74) dirty exists 2014-07-01 16:52:08,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 1 to mira0437250-76 2014-07-01 16:52:08,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 1 to mira0437250-77 2014-07-01 16:52:08,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 3 to mira0437250-76 2014-07-01 16:52:08,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 3 to mira0437250-77 2014-07-01 16:52:08,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: finishing write tid 4 to mira0437250-76 2014-07-01 16:52:08,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 25 (ObjNum 75 snap 0 seq_num 75) dirty exists 2014-07-01 16:52:08,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: finishing write tid 4 to mira0437250-77 2014-07-01 16:52:08,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 20 (ObjNum 76 snap 0 seq_num 76) dirty exists 2014-07-01 16:52:08,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 3 to mira0437250-74 2014-07-01 16:52:08,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 5 to mira0437250-74 2014-07-01 16:52:08,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: finishing write tid 6 to mira0437250-74 2014-07-01 16:52:08,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 22 (ObjNum 73 snap 0 seq_num 73) dirty exists 2014-07-01 16:52:08,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 1 to mira0437250-80 2014-07-01 16:52:08,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 2 to mira0437250-80 2014-07-01 16:52:08,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 4 to mira0437250-80 2014-07-01 16:52:08,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: finishing write tid 5 to mira0437250-80 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 23 (ObjNum 79 snap 0 seq_num 79) dirty exists 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 1 to mira0437250-81 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 2 to mira0437250-81 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 3 to mira0437250-81 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 5 to mira0437250-81 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: finishing write tid 6 to mira0437250-81 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 81 v 29 (ObjNum 80 snap 0 seq_num 80) dirty exists 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 1 to mira0437250-83 2014-07-01 16:52:08,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 1 to mira0437250-84 2014-07-01 16:52:08,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 1 to mira0437250-85 2014-07-01 16:52:08,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 3 to mira0437250-85 2014-07-01 16:52:08,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 3 to mira0437250-84 2014-07-01 16:52:08,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 3 to mira0437250-83 2014-07-01 16:52:08,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: finishing write tid 4 to mira0437250-85 2014-07-01 16:52:08,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 85 v 22 (ObjNum 84 snap 0 seq_num 84) dirty exists 2014-07-01 16:52:08,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: finishing write tid 4 to mira0437250-83 2014-07-01 16:52:08,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 31 (ObjNum 82 snap 0 seq_num 82) dirty exists 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: finishing write tid 4 to mira0437250-84 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 22 (ObjNum 83 snap 0 seq_num 83) dirty exists 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 1 to mira0437250-86 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:73: done (15 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:74: done (14 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:75: done (13 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:76: done (12 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:77: done (11 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:80: done (10 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:81: done (9 left) 2014-07-01 16:52:08,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:83: done (8 left) 2014-07-01 16:52:08,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:84: done (7 left) 2014-07-01 16:52:08,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:85: done (6 left) 2014-07-01 16:52:08,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: write initial oid 87 2014-07-01 16:52:08,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: seq_num 86 ranges {725569=766372,1930304=395265} 2014-07-01 16:52:08,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: writing mira0437250-87 from 725569 to 1491941 tid 1 2014-07-01 16:52:08,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: writing mira0437250-87 from 1930304 to 2325569 tid 2 2014-07-01 16:52:08,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: write initial oid 88 2014-07-01 16:52:08,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: seq_num 87 ranges {421823=739201,1645461=376362} 2014-07-01 16:52:08,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: writing mira0437250-88 from 421823 to 1161024 tid 1 2014-07-01 16:52:08,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: writing mira0437250-88 from 1645461 to 2021823 tid 2 2014-07-01 16:52:08,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: write initial oid 89 2014-07-01 16:52:08,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: seq_num 88 ranges {615229=462429,1724521=513605,2739010=629687,3815228=1} 2014-07-01 16:52:08,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0437250-89 from 615229 to 1077658 tid 1 2014-07-01 16:52:08,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0437250-89 from 1724521 to 2238126 tid 2 2014-07-01 16:52:08,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0437250-89 from 2739010 to 3368697 tid 3 2014-07-01 16:52:08,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: writing mira0437250-89 from 3815228 to 3815229 tid 4 2014-07-01 16:52:08,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: write initial oid 90 2014-07-01 16:52:08,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: seq_num 89 ranges {710458=717881,1969852=583800,3071434=439024} 2014-07-01 16:52:08,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: writing mira0437250-90 from 710458 to 1428339 tid 1 2014-07-01 16:52:08,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: writing mira0437250-90 from 1969852 to 2553652 tid 2 2014-07-01 16:52:08,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: writing mira0437250-90 from 3071434 to 3510458 tid 3 2014-07-01 16:52:08,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: write initial oid 91 2014-07-01 16:52:08,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: seq_num 90 ranges {406712=441110,1435409=653995,2872660=334052} 2014-07-01 16:52:08,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0437250-91 from 406712 to 847822 tid 1 2014-07-01 16:52:09,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0437250-91 from 1435409 to 2089404 tid 2 2014-07-01 16:52:09,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: writing mira0437250-91 from 2872660 to 3206712 tid 3 2014-07-01 16:52:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: write initial oid 92 2014-07-01 16:52:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: seq_num 91 ranges {715446=662514,1515445=1} 2014-07-01 16:52:09,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0437250-92 from 715446 to 1377960 tid 1 2014-07-01 16:52:09,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: writing mira0437250-92 from 1515445 to 1515446 tid 2 2014-07-01 16:52:09,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: write initial oid 93 2014-07-01 16:52:09,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: seq_num 92 ranges {410675=785742,1210674=1} 2014-07-01 16:52:09,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: writing mira0437250-93 from 410675 to 1196417 tid 1 2014-07-01 16:52:09,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: writing mira0437250-93 from 1210674 to 1210675 tid 2 2014-07-01 16:52:09,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: write initial oid 94 2014-07-01 16:52:09,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: seq_num 93 ranges {506929=400000} 2014-07-01 16:52:09,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: writing mira0437250-94 from 506929 to 906929 tid 1 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: write initial oid 95 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: seq_num 94 ranges {450735=765447,1650734=1} 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0437250-95 from 450735 to 1216182 tid 1 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: writing mira0437250-95 from 1650734 to 1650735 tid 2 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: write initial oid 96 2014-07-01 16:52:09,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: seq_num 95 ranges {511917=737251,1995958=790443,2911916=1} 2014-07-01 16:52:09,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0437250-96 from 511917 to 1249168 tid 1 2014-07-01 16:52:09,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0437250-96 from 1995958 to 2786401 tid 2 2014-07-01 16:52:09,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: writing mira0437250-96 from 2911916 to 2911917 tid 3 2014-07-01 16:52:09,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 2 to mira0437250-86 2014-07-01 16:52:09,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 4 to mira0437250-86 2014-07-01 16:52:09,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: finishing write tid 5 to mira0437250-86 2014-07-01 16:52:09,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 14 (ObjNum 85 snap 0 seq_num 85) dirty exists 2014-07-01 16:52:09,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 1 to mira0437250-87 2014-07-01 16:52:09,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 2 to mira0437250-87 2014-07-01 16:52:09,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 4 to mira0437250-87 2014-07-01 16:52:09,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: finishing write tid 5 to mira0437250-87 2014-07-01 16:52:09,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 87 v 14 (ObjNum 86 snap 0 seq_num 86) dirty exists 2014-07-01 16:52:09,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 1 to mira0437250-88 2014-07-01 16:52:09,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 2 to mira0437250-88 2014-07-01 16:52:09,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 4 to mira0437250-88 2014-07-01 16:52:09,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:86: done (15 left) 2014-07-01 16:52:09,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:87: done (14 left) 2014-07-01 16:52:09,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: write initial oid 97 2014-07-01 16:52:09,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: seq_num 96 ranges {607146=460480,1576842=612062,2607145=1} 2014-07-01 16:52:09,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: writing mira0437250-97 from 607146 to 1067626 tid 1 2014-07-01 16:52:09,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: writing mira0437250-97 from 1576842 to 2188904 tid 2 2014-07-01 16:52:09,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: writing mira0437250-97 from 2607145 to 2607146 tid 3 2014-07-01 16:52:09,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: write initial oid 98 2014-07-01 16:52:09,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: seq_num 97 ranges {551976=715932,1823198=530834,2758166=446514,3351975=1} 2014-07-01 16:52:09,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0437250-98 from 551976 to 1267908 tid 1 2014-07-01 16:52:09,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0437250-98 from 1823198 to 2354032 tid 2 2014-07-01 16:52:09,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0437250-98 from 2758166 to 3204680 tid 3 2014-07-01 16:52:09,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: writing mira0437250-98 from 3351975 to 3351976 tid 4 2014-07-01 16:52:09,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: finishing write tid 5 to mira0437250-88 2014-07-01 16:52:09,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 17 (ObjNum 87 snap 0 seq_num 87) dirty exists 2014-07-01 16:52:09,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 1 to mira0437250-89 2014-07-01 16:52:09,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 2 to mira0437250-89 2014-07-01 16:52:09,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 3 to mira0437250-89 2014-07-01 16:52:09,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 4 to mira0437250-89 2014-07-01 16:52:09,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 6 to mira0437250-89 2014-07-01 16:52:09,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: finishing write tid 7 to mira0437250-89 2014-07-01 16:52:09,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 89 v 19 (ObjNum 88 snap 0 seq_num 88) dirty exists 2014-07-01 16:52:09,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 1 to mira0437250-92 2014-07-01 16:52:09,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 2 to mira0437250-92 2014-07-01 16:52:09,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 4 to mira0437250-92 2014-07-01 16:52:09,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: finishing write tid 5 to mira0437250-92 2014-07-01 16:52:09,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 25 (ObjNum 91 snap 0 seq_num 91) dirty exists 2014-07-01 16:52:09,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 1 to mira0437250-94 2014-07-01 16:52:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:88: done (15 left) 2014-07-01 16:52:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:89: done (14 left) 2014-07-01 16:52:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:92: done (13 left) 2014-07-01 16:52:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: write initial oid 99 2014-07-01 16:52:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: seq_num 98 ranges {96805=1} 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: writing mira0437250-99 from 96805 to 96806 tid 1 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: write initial oid 100 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: seq_num 99 ranges {592036=411989,1464967=422648,2325545=598847,3377961=414075} 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0437250-100 from 592036 to 1004025 tid 1 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0437250-100 from 1464967 to 1887615 tid 2 2014-07-01 16:52:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0437250-100 from 2325545 to 2924392 tid 3 2014-07-01 16:52:09,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: writing mira0437250-100 from 3377961 to 3792036 tid 4 2014-07-01 16:52:09,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: write initial oid 101 2014-07-01 16:52:09,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: seq_num 100 ranges {652193=400000} 2014-07-01 16:52:09,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: writing mira0437250-101 from 652193 to 1052193 tid 1 2014-07-01 16:52:09,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 3 to mira0437250-94 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: finishing write tid 4 to mira0437250-94 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 33 (ObjNum 93 snap 0 seq_num 93) dirty exists 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 1 to mira0437250-95 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 2 to mira0437250-95 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 4 to mira0437250-95 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: finishing write tid 5 to mira0437250-95 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 17 (ObjNum 94 snap 0 seq_num 94) dirty exists 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 1 to mira0437250-96 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 2 to mira0437250-96 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 3 to mira0437250-96 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 5 to mira0437250-96 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: finishing write tid 6 to mira0437250-96 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 21 (ObjNum 95 snap 0 seq_num 95) dirty exists 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 1 to mira0437250-97 2014-07-01 16:52:09,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 2 to mira0437250-97 2014-07-01 16:52:09,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 3 to mira0437250-97 2014-07-01 16:52:09,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 5 to mira0437250-97 2014-07-01 16:52:09,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: finishing write tid 6 to mira0437250-97 2014-07-01 16:52:09,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 26 (ObjNum 96 snap 0 seq_num 96) dirty exists 2014-07-01 16:52:09,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 1 to mira0437250-98 2014-07-01 16:52:09,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 2 to mira0437250-98 2014-07-01 16:52:09,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 3 to mira0437250-98 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 1 to mira0437250-99 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 4 to mira0437250-98 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 6 to mira0437250-98 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 3 to mira0437250-99 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: finishing write tid 4 to mira0437250-99 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 99 v 13 (ObjNum 98 snap 0 seq_num 98) dirty exists 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: finishing write tid 7 to mira0437250-98 2014-07-01 16:52:09,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 26 (ObjNum 97 snap 0 seq_num 97) dirty exists 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:94: done (15 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:95: done (14 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:96: done (13 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:97: done (12 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:98: done (11 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:99: done (10 left) 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: write initial oid 102 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: seq_num 101 ranges {597023=791693,1797022=1} 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: writing mira0437250-102 from 597023 to 1388716 tid 1 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: writing mira0437250-102 from 1797022 to 1797023 tid 2 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: write initial oid 103 2014-07-01 16:52:09,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: seq_num 102 ranges {692253=764522,1492252=1} 2014-07-01 16:52:09,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: writing mira0437250-103 from 692253 to 1456775 tid 1 2014-07-01 16:52:09,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: writing mira0437250-103 from 1492252 to 1492253 tid 2 2014-07-01 16:52:09,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: write initial oid 104 2014-07-01 16:52:09,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: seq_num 103 ranges {637083=736326,2098903=138180} 2014-07-01 16:52:09,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: writing mira0437250-104 from 637083 to 1373409 tid 1 2014-07-01 16:52:09,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: writing mira0437250-104 from 2098903 to 2237083 tid 2 2014-07-01 16:52:09,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: write initial oid 105 2014-07-01 16:52:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: seq_num 104 ranges {697240=459554,1776938=560120,2877399=619841} 2014-07-01 16:52:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0437250-105 from 697240 to 1156794 tid 1 2014-07-01 16:52:09,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0437250-105 from 1776938 to 2337058 tid 2 2014-07-01 16:52:09,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: writing mira0437250-105 from 2877399 to 3497240 tid 3 2014-07-01 16:52:09,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: write initial oid 106 2014-07-01 16:52:09,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: seq_num 105 ranges {793494=716031,2175743=630315,3193493=1} 2014-07-01 16:52:09,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: writing mira0437250-106 from 793494 to 1509525 tid 1 2014-07-01 16:52:09,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: writing mira0437250-106 from 2175743 to 2806058 tid 2 2014-07-01 16:52:09,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: writing mira0437250-106 from 3193493 to 3193494 tid 3 2014-07-01 16:52:09,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: write initial oid 107 2014-07-01 16:52:09,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: seq_num 106 ranges {738324=439259,1606227=451934,2481898=795564,3695171=243153} 2014-07-01 16:52:09,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0437250-107 from 738324 to 1177583 tid 1 2014-07-01 16:52:09,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0437250-107 from 1606227 to 2058161 tid 2 2014-07-01 16:52:09,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0437250-107 from 2481898 to 3277462 tid 3 2014-07-01 16:52:09,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: writing mira0437250-107 from 3695171 to 3938324 tid 4 2014-07-01 16:52:09,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 1 to mira0437250-79 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 1 to mira0437250-69 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 2 to mira0437250-79 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 2 to mira0437250-69 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 4 to mira0437250-69 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: finishing write tid 5 to mira0437250-69 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 69 v 13 (ObjNum 68 snap 0 seq_num 68) dirty exists 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 3 to mira0437250-79 2014-07-01 16:52:09,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 1 to mira0437250-70 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 2 to mira0437250-70 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 1 to mira0437250-78 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 4 to mira0437250-70 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:69: done (15 left) 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: write initial oid 108 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: seq_num 107 ranges {433554=411063,1567911=673553,2717171=681331,3633553=1} 2014-07-01 16:52:09,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0437250-108 from 433554 to 844617 tid 1 2014-07-01 16:52:09,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0437250-108 from 1567911 to 2241464 tid 2 2014-07-01 16:52:09,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0437250-108 from 2717171 to 3398502 tid 3 2014-07-01 16:52:09,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: writing mira0437250-108 from 3633553 to 3633554 tid 4 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: finishing write tid 5 to mira0437250-70 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 70 v 16 (ObjNum 69 snap 0 seq_num 69) dirty exists 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 1 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 2 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 3 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 4 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 6 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: finishing write tid 7 to mira0437250-82 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 21 (ObjNum 81 snap 0 seq_num 81) dirty exists 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 1 to mira0437250-90 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:70: done (15 left) 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:82: done (14 left) 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: write initial oid 109 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: seq_num 108 ranges {377358=1} 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: writing mira0437250-109 from 377358 to 377359 tid 1 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: write initial oid 110 2014-07-01 16:52:09,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: seq_num 109 ranges {437517=790768,1637516=1} 2014-07-01 16:52:09,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: writing mira0437250-110 from 437517 to 1228285 tid 1 2014-07-01 16:52:09,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: writing mira0437250-110 from 1637516 to 1637517 tid 2 2014-07-01 16:52:09,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 2 to mira0437250-90 2014-07-01 16:52:09,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 3 to mira0437250-90 2014-07-01 16:52:09,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 5 to mira0437250-90 2014-07-01 16:52:09,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: finishing write tid 6 to mira0437250-90 2014-07-01 16:52:09,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 90 v 25 (ObjNum 89 snap 0 seq_num 89) dirty exists 2014-07-01 16:52:09,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 1 to mira0437250-91 2014-07-01 16:52:09,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 2 to mira0437250-91 2014-07-01 16:52:09,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:90: done (15 left) 2014-07-01 16:52:09,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: write initial oid 111 2014-07-01 16:52:09,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: seq_num 110 ranges {533771=513996,1333770=1} 2014-07-01 16:52:09,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: writing mira0437250-111 from 533771 to 1047767 tid 1 2014-07-01 16:52:09,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: writing mira0437250-111 from 1333770 to 1333771 tid 2 2014-07-01 16:52:09,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 3 to mira0437250-91 2014-07-01 16:52:09,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 5 to mira0437250-91 2014-07-01 16:52:09,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 2 to mira0437250-78 2014-07-01 16:52:09,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: finishing write tid 6 to mira0437250-91 2014-07-01 16:52:09,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 91 v 29 (ObjNum 90 snap 0 seq_num 90) dirty exists 2014-07-01 16:52:09,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 4 to mira0437250-78 2014-07-01 16:52:09,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:91: done (15 left) 2014-07-01 16:52:09,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: write initial oid 112 2014-07-01 16:52:09,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: seq_num 111 ranges {478600=485801,1552248=437982,2078599=1} 2014-07-01 16:52:09,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0437250-112 from 478600 to 964401 tid 1 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0437250-112 from 1552248 to 1990230 tid 2 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: writing mira0437250-112 from 2078599 to 2078600 tid 3 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 1 to mira0437250-93 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 2 to mira0437250-93 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 4 to mira0437250-93 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 1 to mira0437250-100 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 2 to mira0437250-100 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 3 to mira0437250-100 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 1 to mira0437250-101 2014-07-01 16:52:09,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 4 to mira0437250-100 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 6 to mira0437250-100 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 3 to mira0437250-101 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 1 to mira0437250-102 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 2 to mira0437250-102 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 4 to mira0437250-102 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 1 to mira0437250-103 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 2 to mira0437250-103 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 4 to mira0437250-103 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 1 to mira0437250-104 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 2 to mira0437250-104 2014-07-01 16:52:09,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 4 to mira0437250-104 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: finishing write tid 5 to mira0437250-78 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 10 (ObjNum 77 snap 0 seq_num 77) dirty exists 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: finishing write tid 5 to mira0437250-93 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 18 (ObjNum 92 snap 0 seq_num 92) dirty exists 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: finishing write tid 4 to mira0437250-101 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 16 (ObjNum 100 snap 0 seq_num 100) dirty exists 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 5 to mira0437250-79 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:78: done (15 left) 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:93: done (14 left) 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:101: done (13 left) 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: write initial oid 113 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: seq_num 112 ranges {422406=610053,1666380=658577,2822405=1} 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0437250-113 from 422406 to 1032459 tid 1 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0437250-113 from 1666380 to 2324957 tid 2 2014-07-01 16:52:09,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: writing mira0437250-113 from 2822405 to 2822406 tid 3 2014-07-01 16:52:09,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: write initial oid 114 2014-07-01 16:52:09,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: seq_num 113 ranges {635012=714081,1994016=728773,3035011=1} 2014-07-01 16:52:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: writing mira0437250-114 from 635012 to 1349093 tid 1 2014-07-01 16:52:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: writing mira0437250-114 from 1994016 to 2722789 tid 2 2014-07-01 16:52:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: writing mira0437250-114 from 3035011 to 3035012 tid 3 2014-07-01 16:52:09,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: write initial oid 115 2014-07-01 16:52:09,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: seq_num 114 ranges {578818=437310,1707125=550392,2719030=609249,3778817=1} 2014-07-01 16:52:09,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0437250-115 from 578818 to 1016128 tid 1 2014-07-01 16:52:09,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0437250-115 from 1707125 to 2257517 tid 2 2014-07-01 16:52:09,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0437250-115 from 2719030 to 3328279 tid 3 2014-07-01 16:52:09,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: writing mira0437250-115 from 3778817 to 3778818 tid 4 2014-07-01 16:52:10,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: finishing write tid 6 to mira0437250-79 2014-07-01 16:52:10,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 9 (ObjNum 78 snap 0 seq_num 78) dirty exists 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: finishing write tid 7 to mira0437250-100 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 23 (ObjNum 99 snap 0 seq_num 99) dirty exists 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: finishing write tid 5 to mira0437250-102 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 13 (ObjNum 101 snap 0 seq_num 101) dirty exists 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: finishing write tid 5 to mira0437250-103 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 19 (ObjNum 102 snap 0 seq_num 102) dirty exists 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: finishing write tid 5 to mira0437250-104 2014-07-01 16:52:10,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 104 v 16 (ObjNum 103 snap 0 seq_num 103) dirty exists 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 1 to mira0437250-105 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:79: done (15 left) 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:100: done (14 left) 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:102: done (13 left) 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:103: done (12 left) 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:104: done (11 left) 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: write initial oid 116 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: seq_num 115 ranges {523646=1} 2014-07-01 16:52:10,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: writing mira0437250-116 from 523646 to 523647 tid 1 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: write initial oid 117 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: seq_num 116 ranges {219900=1} 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: writing mira0437250-117 from 219900 to 219901 tid 1 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: write initial oid 118 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: seq_num 117 ranges {562683=400000} 2014-07-01 16:52:10,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: writing mira0437250-118 from 562683 to 962683 tid 1 2014-07-01 16:52:10,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: write initial oid 119 2014-07-01 16:52:10,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: seq_num 118 ranges {623865=512047,1691461=532404} 2014-07-01 16:52:10,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: writing mira0437250-119 from 623865 to 1135912 tid 1 2014-07-01 16:52:10,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: writing mira0437250-119 from 1691461 to 2223865 tid 2 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: write initial oid 120 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: seq_num 119 ranges {719094=484876,1805593=113501} 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: writing mira0437250-120 from 719094 to 1203970 tid 1 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: writing mira0437250-120 from 1805593 to 1919094 tid 2 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 1 to mira0437250-106 2014-07-01 16:52:10,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 2 to mira0437250-106 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 3 to mira0437250-106 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 5 to mira0437250-106 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: finishing write tid 6 to mira0437250-106 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 13 (ObjNum 105 snap 0 seq_num 105) dirty exists 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 1 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 2 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 2 to mira0437250-105 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 3 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 4 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 6 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: finishing write tid 7 to mira0437250-107 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 107 v 24 (ObjNum 106 snap 0 seq_num 106) dirty exists 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 3 to mira0437250-105 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 5 to mira0437250-105 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 1 to mira0437250-108 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:106: done (15 left) 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:107: done (14 left) 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: write initial oid 121 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: seq_num 120 ranges {415348=608104,1519725=95623} 2014-07-01 16:52:10,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: writing mira0437250-121 from 415348 to 1023452 tid 1 2014-07-01 16:52:10,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: writing mira0437250-121 from 1519725 to 1615348 tid 2 2014-07-01 16:52:10,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: write initial oid 122 2014-07-01 16:52:10,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: seq_num 121 ranges {608754=463556,1731009=676830,3006583=402171} 2014-07-01 16:52:10,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: writing mira0437250-122 from 608754 to 1072310 tid 1 2014-07-01 16:52:10,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: writing mira0437250-122 from 1731009 to 2407839 tid 2 2014-07-01 16:52:10,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: writing mira0437250-122 from 3006583 to 3408754 tid 3 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 1 to mira0437250-109 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 3 to mira0437250-109 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 2 to mira0437250-108 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: finishing write tid 4 to mira0437250-109 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 35 (ObjNum 108 snap 0 seq_num 108) dirty exists 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 3 to mira0437250-108 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:109: done (15 left) 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: write initial oid 123 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: seq_num 122 ranges {420336=587809,1712918=498449,2959233=539285,3620335=1} 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: writing mira0437250-123 from 420336 to 1008145 tid 1 2014-07-01 16:52:10,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: writing mira0437250-123 from 1712918 to 2211367 tid 2 2014-07-01 16:52:10,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: writing mira0437250-123 from 2959233 to 3498518 tid 3 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: writing mira0437250-123 from 3620335 to 3620336 tid 4 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 4 to mira0437250-108 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 6 to mira0437250-108 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 1 to mira0437250-110 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 2 to mira0437250-110 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 4 to mira0437250-110 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: finishing write tid 5 to mira0437250-110 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 26 (ObjNum 109 snap 0 seq_num 109) dirty exists 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 1 to mira0437250-111 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 2 to mira0437250-111 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 4 to mira0437250-111 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: finishing write tid 5 to mira0437250-111 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 20 (ObjNum 110 snap 0 seq_num 110) dirty exists 2014-07-01 16:52:10,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 1 to mira0437250-112 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 2 to mira0437250-112 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 3 to mira0437250-112 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 5 to mira0437250-112 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: finishing write tid 6 to mira0437250-112 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 112 v 27 (ObjNum 111 snap 0 seq_num 111) dirty exists 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 1 to mira0437250-113 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 2 to mira0437250-113 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 3 to mira0437250-113 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:110: done (15 left) 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:111: done (14 left) 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:112: done (13 left) 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: write initial oid 124 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: seq_num 123 ranges {365164=1} 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: writing mira0437250-124 from 365164 to 365165 tid 1 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: write initial oid 125 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: seq_num 124 ranges {708971=400000} 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: writing mira0437250-125 from 708971 to 1108971 tid 1 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: write initial oid 126 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: seq_num 125 ranges {404201=400000} 2014-07-01 16:52:10,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: writing mira0437250-126 from 404201 to 804201 tid 1 2014-07-01 16:52:10,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 5 to mira0437250-113 2014-07-01 16:52:10,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: finishing write tid 6 to mira0437250-113 2014-07-01 16:52:10,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 113 v 20 (ObjNum 112 snap 0 seq_num 112) dirty exists 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 1 to mira0437250-114 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 2 to mira0437250-114 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 3 to mira0437250-114 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 5 to mira0437250-114 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: finishing write tid 6 to mira0437250-114 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 24 (ObjNum 113 snap 0 seq_num 113) dirty exists 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 1 to mira0437250-115 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:113: done (15 left) 2014-07-01 16:52:10,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:114: done (14 left) 2014-07-01 16:52:10,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: write initial oid 127 2014-07-01 16:52:10,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: seq_num 126 ranges {749031=511122,1549030=1} 2014-07-01 16:52:10,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: writing mira0437250-127 from 749031 to 1260153 tid 1 2014-07-01 16:52:10,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: writing mira0437250-127 from 1549030 to 1549031 tid 2 2014-07-01 16:52:10,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: write initial oid 128 2014-07-01 16:52:10,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: seq_num 127 ranges {409188=482926,1356090=635921,2710063=99125} 2014-07-01 16:52:10,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0437250-128 from 409188 to 892114 tid 1 2014-07-01 16:52:10,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0437250-128 from 1356090 to 1992011 tid 2 2014-07-01 16:52:10,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: writing mira0437250-128 from 2710063 to 2809188 tid 3 2014-07-01 16:52:10,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 2 to mira0437250-115 2014-07-01 16:52:10,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 1 to mira0437250-116 2014-07-01 16:52:10,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 3 to mira0437250-116 2014-07-01 16:52:10,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: finishing write tid 4 to mira0437250-116 2014-07-01 16:52:10,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 28 (ObjNum 115 snap 0 seq_num 115) dirty exists 2014-07-01 16:52:10,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 3 to mira0437250-115 2014-07-01 16:52:10,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 4 to mira0437250-115 2014-07-01 16:52:10,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 1 to mira0437250-117 2014-07-01 16:52:10,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 6 to mira0437250-115 2014-07-01 16:52:10,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 3 to mira0437250-117 2014-07-01 16:52:10,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:116: done (15 left) 2014-07-01 16:52:10,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: write initial oid 129 2014-07-01 16:52:10,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: seq_num 128 ranges {505442=606155,1737999=706116,2505441=1} 2014-07-01 16:52:10,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: writing mira0437250-129 from 505442 to 1111597 tid 1 2014-07-01 16:52:10,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: writing mira0437250-129 from 1737999 to 2444115 tid 2 2014-07-01 16:52:10,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: writing mira0437250-129 from 2505441 to 2505442 tid 3 2014-07-01 16:52:10,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: finishing write tid 4 to mira0437250-117 2014-07-01 16:52:10,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 117 v 16 (ObjNum 116 snap 0 seq_num 116) dirty exists 2014-07-01 16:52:10,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: finishing write tid 7 to mira0437250-115 2014-07-01 16:52:10,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 18 (ObjNum 114 snap 0 seq_num 114) dirty exists 2014-07-01 16:52:10,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 1 to mira0437250-118 2014-07-01 16:52:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 3 to mira0437250-118 2014-07-01 16:52:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: finishing write tid 4 to mira0437250-118 2014-07-01 16:52:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 22 (ObjNum 117 snap 0 seq_num 117) dirty exists 2014-07-01 16:52:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 1 to mira0437250-119 2014-07-01 16:52:10,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 2 to mira0437250-119 2014-07-01 16:52:10,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 4 to mira0437250-119 2014-07-01 16:52:10,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:115: done (15 left) 2014-07-01 16:52:10,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:117: done (14 left) 2014-07-01 16:52:10,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:118: done (13 left) 2014-07-01 16:52:10,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: write initial oid 130 2014-07-01 16:52:10,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: seq_num 129 ranges {449248=461607,1680483=776312,2940867=308381} 2014-07-01 16:52:10,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: writing mira0437250-130 from 449248 to 910855 tid 1 2014-07-01 16:52:10,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: writing mira0437250-130 from 1680483 to 2456795 tid 2 2014-07-01 16:52:10,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: writing mira0437250-130 from 2940867 to 3249248 tid 3 2014-07-01 16:52:10,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: write initial oid 131 2014-07-01 16:52:10,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: seq_num 130 ranges {794077=585859,1795638=597931,3144139=753993,3994076=1} 2014-07-01 16:52:10,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0437250-131 from 794077 to 1379936 tid 1 2014-07-01 16:52:10,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0437250-131 from 1795638 to 2393569 tid 2 2014-07-01 16:52:10,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0437250-131 from 3144139 to 3898132 tid 3 2014-07-01 16:52:10,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: writing mira0437250-131 from 3994076 to 3994077 tid 4 2014-07-01 16:52:10,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: write initial oid 132 2014-07-01 16:52:10,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: seq_num 131 ranges {206682=1} 2014-07-01 16:52:10,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: writing mira0437250-132 from 206682 to 206683 tid 1 2014-07-01 16:52:10,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: finishing write tid 5 to mira0437250-119 2014-07-01 16:52:10,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 29 (ObjNum 118 snap 0 seq_num 118) dirty exists 2014-07-01 16:52:10,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: finishing write tid 6 to mira0437250-105 2014-07-01 16:52:10,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 29 (ObjNum 104 snap 0 seq_num 104) dirty exists 2014-07-01 16:52:10,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: finishing write tid 7 to mira0437250-108 2014-07-01 16:52:10,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 34 (ObjNum 107 snap 0 seq_num 107) dirty exists 2014-07-01 16:52:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 1 to mira0437250-120 2014-07-01 16:52:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:105: done (15 left) 2014-07-01 16:52:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:108: done (14 left) 2014-07-01 16:52:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:119: done (13 left) 2014-07-01 16:52:10,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: write initial oid 133 2014-07-01 16:52:10,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: seq_num 132 ranges {550489=400000} 2014-07-01 16:52:10,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: writing mira0437250-133 from 550489 to 950489 tid 1 2014-07-01 16:52:10,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: write initial oid 134 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: seq_num 133 ranges {645718=1} 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: writing mira0437250-134 from 645718 to 645719 tid 1 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: write initial oid 135 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: seq_num 134 ranges {589524=661621,1389523=1} 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: writing mira0437250-135 from 589524 to 1251145 tid 1 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: writing mira0437250-135 from 1389523 to 1389524 tid 2 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 2 to mira0437250-120 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 4 to mira0437250-120 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: finishing write tid 5 to mira0437250-120 2014-07-01 16:52:10,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 16 (ObjNum 119 snap 0 seq_num 119) dirty exists 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 1 to mira0437250-121 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 2 to mira0437250-121 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 4 to mira0437250-121 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:120: done (15 left) 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: write initial oid 136 2014-07-01 16:52:10,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: seq_num 135 ranges {534354=482001,1591259=543095} 2014-07-01 16:52:10,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: writing mira0437250-136 from 534354 to 1016355 tid 1 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: writing mira0437250-136 from 1591259 to 2134354 tid 2 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: finishing write tid 5 to mira0437250-121 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 25 (ObjNum 120 snap 0 seq_num 120) dirty exists 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 1 to mira0437250-122 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 1 to mira0437250-124 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 3 to mira0437250-124 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: finishing write tid 4 to mira0437250-124 2014-07-01 16:52:10,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 31 (ObjNum 123 snap 0 seq_num 123) dirty exists 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 1 to mira0437250-127 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 2 to mira0437250-127 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 4 to mira0437250-127 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: finishing write tid 5 to mira0437250-127 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 19 (ObjNum 126 snap 0 seq_num 126) dirty exists 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 1 to mira0437250-129 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 2 to mira0437250-129 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 3 to mira0437250-129 2014-07-01 16:52:10,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 5 to mira0437250-129 2014-07-01 16:52:10,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: finishing write tid 6 to mira0437250-129 2014-07-01 16:52:10,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 23 (ObjNum 128 snap 0 seq_num 128) dirty exists 2014-07-01 16:52:10,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 1 to mira0437250-130 2014-07-01 16:52:10,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 2 to mira0437250-130 2014-07-01 16:52:10,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 3 to mira0437250-130 2014-07-01 16:52:10,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 5 to mira0437250-130 2014-07-01 16:52:10,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: finishing write tid 6 to mira0437250-130 2014-07-01 16:52:10,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 29 (ObjNum 129 snap 0 seq_num 129) dirty exists 2014-07-01 16:52:10,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 1 to mira0437250-131 2014-07-01 16:52:10,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 2 to mira0437250-131 2014-07-01 16:52:10,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 1 to mira0437250-132 2014-07-01 16:52:10,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 3 to mira0437250-132 2014-07-01 16:52:10,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: finishing write tid 4 to mira0437250-132 2014-07-01 16:52:10,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 132 v 28 (ObjNum 131 snap 0 seq_num 131) dirty exists 2014-07-01 16:52:10,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 3 to mira0437250-131 2014-07-01 16:52:10,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 4 to mira0437250-131 2014-07-01 16:52:10,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 6 to mira0437250-131 2014-07-01 16:52:10,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: finishing write tid 7 to mira0437250-131 2014-07-01 16:52:10,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 28 (ObjNum 130 snap 0 seq_num 130) dirty exists 2014-07-01 16:52:10,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 1 to mira0437250-134 2014-07-01 16:52:10,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 3 to mira0437250-134 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: finishing write tid 4 to mira0437250-134 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 29 (ObjNum 133 snap 0 seq_num 133) dirty exists 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 1 to mira0437250-136 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:121: done (15 left) 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:124: done (14 left) 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:127: done (13 left) 2014-07-01 16:52:10,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:129: done (12 left) 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:130: done (11 left) 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:131: done (10 left) 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:132: done (9 left) 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:134: done (8 left) 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: write initial oid 137 2014-07-01 16:52:10,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: seq_num 136 ranges {746960=605229,2089519=257441} 2014-07-01 16:52:10,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: writing mira0437250-137 from 746960 to 1352189 tid 1 2014-07-01 16:52:10,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: writing mira0437250-137 from 2089519 to 2346960 tid 2 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: write initial oid 138 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: seq_num 137 ranges {690766=461706,1935876=474769,3090765=1} 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: writing mira0437250-138 from 690766 to 1152472 tid 1 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: writing mira0437250-138 from 1935876 to 2410645 tid 2 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: writing mira0437250-138 from 3090765 to 3090766 tid 3 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: write initial oid 139 2014-07-01 16:52:10,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: seq_num 138 ranges {635595=584934,2014935=696388,3499670=335925} 2014-07-01 16:52:10,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: writing mira0437250-139 from 635595 to 1220529 tid 1 2014-07-01 16:52:10,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: writing mira0437250-139 from 2014935 to 2711323 tid 2 2014-07-01 16:52:10,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: writing mira0437250-139 from 3499670 to 3835595 tid 3 2014-07-01 16:52:10,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: write initial oid 140 2014-07-01 16:52:10,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: seq_num 139 ranges {730825=556738,1727019=766583,2898847=419396,3530824=1} 2014-07-01 16:52:10,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0437250-140 from 730825 to 1287563 tid 1 2014-07-01 16:52:10,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0437250-140 from 1727019 to 2493602 tid 2 2014-07-01 16:52:10,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0437250-140 from 2898847 to 3318243 tid 3 2014-07-01 16:52:10,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: writing mira0437250-140 from 3530824 to 3530825 tid 4 2014-07-01 16:52:10,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: write initial oid 141 2014-07-01 16:52:10,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: seq_num 140 ranges {792006=1} 2014-07-01 16:52:10,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: writing mira0437250-141 from 792006 to 792007 tid 1 2014-07-01 16:52:10,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: write initial oid 142 2014-07-01 16:52:10,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: seq_num 141 ranges {735813=536443,1535812=1} 2014-07-01 16:52:10,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: writing mira0437250-142 from 735813 to 1272256 tid 1 2014-07-01 16:52:10,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: writing mira0437250-142 from 1535812 to 1535813 tid 2 2014-07-01 16:52:10,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: write initial oid 143 2014-07-01 16:52:10,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: seq_num 142 ranges {431042=659671,1231041=1} 2014-07-01 16:52:10,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: writing mira0437250-143 from 431042 to 1090713 tid 1 2014-07-01 16:52:10,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: writing mira0437250-143 from 1231041 to 1231042 tid 2 2014-07-01 16:52:10,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: write initial oid 144 2014-07-01 16:52:10,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: seq_num 143 ranges {527296=400000} 2014-07-01 16:52:10,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: writing mira0437250-144 from 527296 to 927296 tid 1 2014-07-01 16:52:10,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 2 to mira0437250-136 2014-07-01 16:52:10,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 4 to mira0437250-136 2014-07-01 16:52:10,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: finishing write tid 5 to mira0437250-136 2014-07-01 16:52:10,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 32 (ObjNum 135 snap 0 seq_num 135) dirty exists 2014-07-01 16:52:10,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 1 to mira0437250-137 2014-07-01 16:52:10,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 2 to mira0437250-137 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 4 to mira0437250-137 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: finishing write tid 5 to mira0437250-137 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 31 (ObjNum 136 snap 0 seq_num 136) dirty exists 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 1 to mira0437250-138 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 2 to mira0437250-138 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 3 to mira0437250-138 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 5 to mira0437250-138 2014-07-01 16:52:10,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: finishing write tid 6 to mira0437250-138 2014-07-01 16:52:10,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 138 v 28 (ObjNum 137 snap 0 seq_num 137) dirty exists 2014-07-01 16:52:10,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 1 to mira0437250-140 2014-07-01 16:52:10,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 2 to mira0437250-140 2014-07-01 16:52:10,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 1 to mira0437250-141 2014-07-01 16:52:10,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 3 to mira0437250-141 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 3 to mira0437250-140 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 4 to mira0437250-140 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: finishing write tid 4 to mira0437250-141 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 30 (ObjNum 140 snap 0 seq_num 140) dirty exists 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 6 to mira0437250-140 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: finishing write tid 7 to mira0437250-140 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 33 (ObjNum 139 snap 0 seq_num 139) dirty exists 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:136: done (15 left) 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:137: done (14 left) 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:138: done (13 left) 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:140: done (12 left) 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:141: done (11 left) 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: write initial oid 145 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: seq_num 144 ranges {719678=755728,2075089=620407,2719677=1} 2014-07-01 16:52:10,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0437250-145 from 719678 to 1475406 tid 1 2014-07-01 16:52:10,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0437250-145 from 2075089 to 2695496 tid 2 2014-07-01 16:52:10,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: writing mira0437250-145 from 2719677 to 2719678 tid 3 2014-07-01 16:52:10,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: write initial oid 146 2014-07-01 16:52:10,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: seq_num 145 ranges {532284=727532,1906597=574251,2932283=1} 2014-07-01 16:52:10,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: writing mira0437250-146 from 532284 to 1259816 tid 1 2014-07-01 16:52:10,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: writing mira0437250-146 from 1906597 to 2480848 tid 2 2014-07-01 16:52:10,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: writing mira0437250-146 from 2932283 to 2932284 tid 3 2014-07-01 16:52:10,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: write initial oid 147 2014-07-01 16:52:10,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: seq_num 146 ranges {628537=582985,1620729=644446,2628536=1} 2014-07-01 16:52:10,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0437250-147 from 628537 to 1211522 tid 1 2014-07-01 16:52:10,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0437250-147 from 1620729 to 2265175 tid 2 2014-07-01 16:52:10,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: writing mira0437250-147 from 2628536 to 2628537 tid 3 2014-07-01 16:52:10,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: write initial oid 148 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: seq_num 147 ranges {420918=1} 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: writing mira0437250-148 from 420918 to 420919 tid 1 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: write initial oid 149 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: seq_num 148 ranges {117172=1} 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: writing mira0437250-149 from 117172 to 117173 tid 1 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:10,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 1 to mira0437250-144 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 3 to mira0437250-144 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: finishing write tid 4 to mira0437250-144 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 30 (ObjNum 143 snap 0 seq_num 143) dirty exists 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 1 to mira0437250-146 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 1 to mira0437250-145 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 2 to mira0437250-145 2014-07-01 16:52:10,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 3 to mira0437250-145 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 5 to mira0437250-145 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 2 to mira0437250-146 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 3 to mira0437250-146 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 5 to mira0437250-146 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: finishing write tid 6 to mira0437250-146 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 34 (ObjNum 145 snap 0 seq_num 145) dirty exists 2014-07-01 16:52:10,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: finishing write tid 6 to mira0437250-145 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 36 (ObjNum 144 snap 0 seq_num 144) dirty exists 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:144: done (15 left) 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:145: done (14 left) 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:146: done (13 left) 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: write initial oid 150 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: seq_num 149 ranges {327729=1} 2014-07-01 16:52:11,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: writing mira0437250-150 from 327729 to 327730 tid 1 2014-07-01 16:52:11,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: write initial oid 151 2014-07-01 16:52:11,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: seq_num 150 ranges {521136=657722,1584841=428075,2121135=1} 2014-07-01 16:52:11,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: writing mira0437250-151 from 521136 to 1178858 tid 1 2014-07-01 16:52:11,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: writing mira0437250-151 from 1584841 to 2012916 tid 2 2014-07-01 16:52:11,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: writing mira0437250-151 from 2121135 to 2121136 tid 3 2014-07-01 16:52:11,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: write initial oid 152 2014-07-01 16:52:11,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: seq_num 151 ranges {617390=630551,1816350=1040} 2014-07-01 16:52:11,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: writing mira0437250-152 from 617390 to 1247941 tid 1 2014-07-01 16:52:11,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: writing mira0437250-152 from 1816350 to 1817390 tid 2 2014-07-01 16:52:11,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 1 to mira0437250-149 2014-07-01 16:52:11,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 3 to mira0437250-149 2014-07-01 16:52:11,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: finishing write tid 4 to mira0437250-149 2014-07-01 16:52:11,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 24 (ObjNum 148 snap 0 seq_num 148) dirty exists 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 1 to mira0437250-150 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 3 to mira0437250-150 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: finishing write tid 4 to mira0437250-150 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 150 v 26 (ObjNum 149 snap 0 seq_num 149) dirty exists 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 1 to mira0437250-151 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 2 to mira0437250-151 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 3 to mira0437250-151 2014-07-01 16:52:11,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 5 to mira0437250-151 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:149: done (15 left) 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:150: done (14 left) 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: write initial oid 153 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: seq_num 152 ranges {712620=753779,1512619=1} 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: writing mira0437250-153 from 712620 to 1466399 tid 1 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: writing mira0437250-153 from 1512619 to 1512620 tid 2 2014-07-01 16:52:11,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: write initial oid 154 2014-07-01 16:52:11,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: seq_num 153 ranges {506025=725583,1857093=789060,3057058=248967} 2014-07-01 16:52:11,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: writing mira0437250-154 from 506025 to 1231608 tid 1 2014-07-01 16:52:11,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: writing mira0437250-154 from 1857093 to 2646153 tid 2 2014-07-01 16:52:11,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: writing mira0437250-154 from 3057058 to 3306025 tid 3 2014-07-01 16:52:11,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: finishing write tid 6 to mira0437250-151 2014-07-01 16:52:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 35 (ObjNum 150 snap 0 seq_num 150) dirty exists 2014-07-01 16:52:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:151: done (15 left) 2014-07-01 16:52:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: write initial oid 155 2014-07-01 16:52:11,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: seq_num 154 ranges {717607=733484,2122650=742903,3294381=223226} 2014-07-01 16:52:11,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: writing mira0437250-155 from 717607 to 1451091 tid 1 2014-07-01 16:52:11,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: writing mira0437250-155 from 2122650 to 2865553 tid 2 2014-07-01 16:52:11,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: writing mira0437250-155 from 3294381 to 3517607 tid 3 2014-07-01 16:52:11,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 1 to mira0437250-152 2014-07-01 16:52:11,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 2 to mira0437250-152 2014-07-01 16:52:11,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 4 to mira0437250-152 2014-07-01 16:52:11,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: finishing write tid 5 to mira0437250-152 2014-07-01 16:52:11,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 27 (ObjNum 151 snap 0 seq_num 151) dirty exists 2014-07-01 16:52:11,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 1 to mira0437250-153 2014-07-01 16:52:11,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 2 to mira0437250-153 2014-07-01 16:52:11,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 4 to mira0437250-153 2014-07-01 16:52:11,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: finishing write tid 5 to mira0437250-153 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 33 (ObjNum 152 snap 0 seq_num 152) dirty exists 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 1 to mira0437250-155 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:152: done (15 left) 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:153: done (14 left) 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: write initial oid 156 2014-07-01 16:52:11,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: seq_num 155 ranges {413861=705288,1835758=564522,3129654=84207} 2014-07-01 16:52:11,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0437250-156 from 413861 to 1119149 tid 1 2014-07-01 16:52:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0437250-156 from 1835758 to 2400280 tid 2 2014-07-01 16:52:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: writing mira0437250-156 from 3129654 to 3213861 tid 3 2014-07-01 16:52:11,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: write initial oid 157 2014-07-01 16:52:11,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: seq_num 156 ranges {758691=677092,1914818=483294,3144384=449907,3958690=1} 2014-07-01 16:52:11,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0437250-157 from 758691 to 1435783 tid 1 2014-07-01 16:52:11,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0437250-157 from 1914818 to 2398112 tid 2 2014-07-01 16:52:11,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0437250-157 from 3144384 to 3594291 tid 3 2014-07-01 16:52:11,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: writing mira0437250-157 from 3958690 to 3958691 tid 4 2014-07-01 16:52:11,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 2 to mira0437250-155 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 3 to mira0437250-155 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 5 to mira0437250-155 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: finishing write tid 6 to mira0437250-155 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 33 (ObjNum 154 snap 0 seq_num 154) dirty exists 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 1 to mira0437250-156 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:155: done (15 left) 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: write initial oid 158 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: seq_num 157 ranges {701471=1} 2014-07-01 16:52:11,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: writing mira0437250-158 from 701471 to 701472 tid 1 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 2 to mira0437250-156 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 3 to mira0437250-156 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 5 to mira0437250-156 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: finishing write tid 6 to mira0437250-156 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 38 (ObjNum 155 snap 0 seq_num 155) dirty exists 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 1 to mira0437250-157 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 2 to mira0437250-157 2014-07-01 16:52:11,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:156: done (15 left) 2014-07-01 16:52:11,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: write initial oid 159 2014-07-01 16:52:11,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: seq_num 158 ranges {514078=400000} 2014-07-01 16:52:11,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: writing mira0437250-159 from 514078 to 914078 tid 1 2014-07-01 16:52:11,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 3 to mira0437250-157 2014-07-01 16:52:11,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 4 to mira0437250-157 2014-07-01 16:52:11,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 1 to mira0437250-158 2014-07-01 16:52:11,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 6 to mira0437250-157 2014-07-01 16:52:11,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 3 to mira0437250-158 2014-07-01 16:52:11,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: finishing write tid 7 to mira0437250-157 2014-07-01 16:52:11,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 40 (ObjNum 156 snap 0 seq_num 156) dirty exists 2014-07-01 16:52:11,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: finishing write tid 4 to mira0437250-158 2014-07-01 16:52:11,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 158 v 24 (ObjNum 157 snap 0 seq_num 157) dirty exists 2014-07-01 16:52:11,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:157: done (15 left) 2014-07-01 16:52:11,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:158: done (14 left) 2014-07-01 16:52:11,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: write initial oid 160 2014-07-01 16:52:11,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: seq_num 159 ranges {457884=629625,1518271=139613} 2014-07-01 16:52:11,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: writing mira0437250-160 from 457884 to 1087509 tid 1 2014-07-01 16:52:11,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: writing mira0437250-160 from 1518271 to 1657884 tid 2 2014-07-01 16:52:11,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: write initial oid 161 2014-07-01 16:52:11,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: seq_num 160 ranges {402714=752854,1597332=666922,2402713=1} 2014-07-01 16:52:11,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: writing mira0437250-161 from 402714 to 1155568 tid 1 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: writing mira0437250-161 from 1597332 to 2264254 tid 2 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: writing mira0437250-161 from 2402713 to 2402714 tid 3 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 1 to mira0437250-160 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 2 to mira0437250-160 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 4 to mira0437250-160 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: finishing write tid 5 to mira0437250-160 2014-07-01 16:52:11,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 39 (ObjNum 159 snap 0 seq_num 159) dirty exists 2014-07-01 16:52:11,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 1 to mira0437250-161 2014-07-01 16:52:11,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:160: done (15 left) 2014-07-01 16:52:11,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: write initial oid 162 2014-07-01 16:52:11,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: seq_num 161 ranges {498967=725682,1712487=386480} 2014-07-01 16:52:11,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0437250-162 from 498967 to 1224649 tid 1 2014-07-01 16:52:11,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: writing mira0437250-162 from 1712487 to 2098967 tid 2 2014-07-01 16:52:11,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 2 to mira0437250-161 2014-07-01 16:52:11,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 3 to mira0437250-161 2014-07-01 16:52:11,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 5 to mira0437250-161 2014-07-01 16:52:11,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: finishing write tid 6 to mira0437250-161 2014-07-01 16:52:11,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 44 (ObjNum 160 snap 0 seq_num 160) dirty exists 2014-07-01 16:52:11,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 1 to mira0437250-162 2014-07-01 16:52:11,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:161: done (15 left) 2014-07-01 16:52:11,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: write initial oid 163 2014-07-01 16:52:11,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: seq_num 162 ranges {442773=732558,1709243=558737,2842772=1} 2014-07-01 16:52:11,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0437250-163 from 442773 to 1175331 tid 1 2014-07-01 16:52:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0437250-163 from 1709243 to 2267980 tid 2 2014-07-01 16:52:11,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: writing mira0437250-163 from 2842772 to 2842773 tid 3 2014-07-01 16:52:11,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: finishing write tid 1 to mira0435601-442 2014-07-01 16:52:11,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: finishing write tid 2 to mira0435601-442 2014-07-01 16:52:11,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: finishing write tid 4 to mira0435601-442 2014-07-01 16:52:11,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: finishing write tid 5 to mira0435601-442 2014-07-01 16:52:11,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 264 (ObjNum 1559 snap 0 seq_num 1559) dirty exists 2014-07-01 16:52:11,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: expect (ObjNum 1153 snap 0 seq_num 1153) 2014-07-01 16:52:11,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 2 to mira0437250-122 2014-07-01 16:52:11,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 3 to mira0437250-122 2014-07-01 16:52:11,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 5 to mira0437250-122 2014-07-01 16:52:11,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: finishing write tid 6 to mira0437250-122 2014-07-01 16:52:11,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 20 (ObjNum 121 snap 0 seq_num 121) dirty exists 2014-07-01 16:52:11,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:122: done (15 left) 2014-07-01 16:52:11,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: write initial oid 164 2014-07-01 16:52:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: seq_num 163 ranges {103954=1} 2014-07-01 16:52:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: writing mira0437250-164 from 103954 to 103955 tid 1 2014-07-01 16:52:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:11,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 1 to mira0437250-125 2014-07-01 16:52:11,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 3 to mira0437250-125 2014-07-01 16:52:12,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 1 to mira0437250-123 2014-07-01 16:52:12,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 2 to mira0437250-123 2014-07-01 16:52:12,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: finishing write tid 4 to mira0437250-125 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 36 (ObjNum 124 snap 0 seq_num 124) dirty exists 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:125: done (15 left) 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: write initial oid 165 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: seq_num 164 ranges {600209=427591,1617764=582775,2833163=664615,3800208=1} 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0437250-165 from 600209 to 1027800 tid 1 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0437250-165 from 1617764 to 2200539 tid 2 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0437250-165 from 2833163 to 3497778 tid 3 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: writing mira0437250-165 from 3800208 to 3800209 tid 4 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 1 to mira0437250-126 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 3 to mira0437250-123 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 3 to mira0437250-126 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: finishing write tid 4 to mira0437250-126 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 38 (ObjNum 125 snap 0 seq_num 125) dirty exists 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 4 to mira0437250-123 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 1 to mira0437250-135 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 2 to mira0437250-135 2014-07-01 16:52:12,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 6 to mira0437250-123 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 1 to mira0437250-128 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 4 to mira0437250-135 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: finishing write tid 5 to mira0437250-135 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 135 v 41 (ObjNum 134 snap 0 seq_num 134) dirty exists 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 2 to mira0437250-128 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 1 to mira0437250-159 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 3 to mira0437250-159 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 3 to mira0437250-128 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: finishing write tid 4 to mira0437250-159 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 159 v 43 (ObjNum 158 snap 0 seq_num 158) dirty exists 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 1 to mira0437250-133 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 5 to mira0437250-128 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 3 to mira0437250-133 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 1 to mira0437250-139 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 2 to mira0437250-139 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 3 to mira0437250-139 2014-07-01 16:52:12,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: finishing write tid 4 to mira0437250-133 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 31 (ObjNum 132 snap 0 seq_num 132) dirty exists 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 5 to mira0437250-139 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: finishing write tid 6 to mira0437250-139 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 139 v 26 (ObjNum 138 snap 0 seq_num 138) dirty exists 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: finishing write tid 7 to mira0437250-123 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 24 (ObjNum 122 snap 0 seq_num 122) dirty exists 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: finishing write tid 6 to mira0437250-128 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 22 (ObjNum 127 snap 0 seq_num 127) dirty exists 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 1 to mira0437250-142 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 2 to mira0437250-142 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 4 to mira0437250-142 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: finishing write tid 5 to mira0437250-142 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 142 v 27 (ObjNum 141 snap 0 seq_num 141) dirty exists 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 1 to mira0437250-143 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 2 to mira0437250-143 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 4 to mira0437250-143 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: finishing write tid 5 to mira0437250-143 2014-07-01 16:52:12,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 34 (ObjNum 142 snap 0 seq_num 142) dirty exists 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 1 to mira0437250-147 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:123: done (15 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:126: done (14 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:128: done (13 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:133: done (12 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:135: done (11 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:139: done (10 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:142: done (9 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:143: done (8 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:159: done (7 left) 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: write initial oid 166 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: seq_num 165 ranges {542989=1} 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: writing mira0437250-166 from 542989 to 542990 tid 1 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: write initial oid 167 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: seq_num 166 ranges {487820=407296,1287819=1} 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: writing mira0437250-167 from 487820 to 895116 tid 1 2014-07-01 16:52:12,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: writing mira0437250-167 from 1287819 to 1287820 tid 2 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: write initial oid 168 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: seq_num 167 ranges {547978=627676,1868768=544785,2547977=1} 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0437250-168 from 547978 to 1175654 tid 1 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0437250-168 from 1868768 to 2413553 tid 2 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: writing mira0437250-168 from 2547977 to 2547978 tid 3 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: write initial oid 169 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: seq_num 168 ranges {644231=750904,1850675=393556} 2014-07-01 16:52:12,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: writing mira0437250-169 from 644231 to 1395135 tid 1 2014-07-01 16:52:12,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: writing mira0437250-169 from 1850675 to 2244231 tid 2 2014-07-01 16:52:12,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: write initial oid 170 2014-07-01 16:52:12,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: seq_num 169 ranges {740485=475157,1940484=1} 2014-07-01 16:52:12,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0437250-170 from 740485 to 1215642 tid 1 2014-07-01 16:52:12,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: writing mira0437250-170 from 1940484 to 1940485 tid 2 2014-07-01 16:52:12,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: write initial oid 171 2014-07-01 16:52:12,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: seq_num 170 ranges {532867=730609,1775068=506794,2750146=725966,3732866=1} 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: writing mira0437250-171 from 532867 to 1263476 tid 1 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: writing mira0437250-171 from 1775068 to 2281862 tid 2 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: writing mira0437250-171 from 2750146 to 3476112 tid 3 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: writing mira0437250-171 from 3732866 to 3732867 tid 4 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: write initial oid 172 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: seq_num 171 ranges {629121=453837,1640624=728413,2987467=441654} 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0437250-172 from 629121 to 1082958 tid 1 2014-07-01 16:52:12,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0437250-172 from 1640624 to 2369037 tid 2 2014-07-01 16:52:12,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: writing mira0437250-172 from 2987467 to 3429121 tid 3 2014-07-01 16:52:12,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: write initial oid 173 2014-07-01 16:52:12,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: seq_num 172 ranges {440702=426666,1319684=432657,2539094=630747,3640701=1} 2014-07-01 16:52:12,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0437250-173 from 440702 to 867368 tid 1 2014-07-01 16:52:12,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0437250-173 from 1319684 to 1752341 tid 2 2014-07-01 16:52:12,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0437250-173 from 2539094 to 3169841 tid 3 2014-07-01 16:52:12,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: writing mira0437250-173 from 3640701 to 3640702 tid 4 2014-07-01 16:52:12,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: write initial oid 174 2014-07-01 16:52:12,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: seq_num 173 ranges {634108=549894,1434107=1} 2014-07-01 16:52:12,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: writing mira0437250-174 from 634108 to 1184002 tid 1 2014-07-01 16:52:12,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: writing mira0437250-174 from 1434107 to 1434108 tid 2 2014-07-01 16:52:12,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 2 to mira0437250-147 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 3 to mira0437250-147 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 5 to mira0437250-147 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: finishing write tid 6 to mira0437250-147 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 38 (ObjNum 146 snap 0 seq_num 146) dirty exists 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 1 to mira0437250-148 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 1 to mira0437250-154 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 3 to mira0437250-148 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 2 to mira0437250-154 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 3 to mira0437250-154 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:147: done (15 left) 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: write initial oid 175 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: seq_num 174 ranges {729338=400000} 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: writing mira0437250-175 from 729338 to 1129338 tid 1 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 5 to mira0437250-154 2014-07-01 16:52:12,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: finishing write tid 4 to mira0437250-148 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 40 (ObjNum 147 snap 0 seq_num 147) dirty exists 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: finishing write tid 6 to mira0437250-154 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 24 (ObjNum 153 snap 0 seq_num 153) dirty exists 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:148: done (15 left) 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:154: done (14 left) 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: write initial oid 176 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: seq_num 175 ranges {424568=400000} 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: writing mira0437250-176 from 424568 to 824568 tid 1 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: write initial oid 177 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: seq_num 176 ranges {734325=501403,1953621=464861,3007099=127226} 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0437250-177 from 734325 to 1235728 tid 1 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0437250-177 from 1953621 to 2418482 tid 2 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: writing mira0437250-177 from 3007099 to 3134325 tid 3 2014-07-01 16:52:12,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:13,853.853 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:13,854.854 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:52:14,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 2 to mira0437250-162 2014-07-01 16:52:14,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 1 to mira0437250-164 2014-07-01 16:52:14,478.478 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:52:14,497.497 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:52:14,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: finishing write tid 1 to mira0435601-423 2014-07-01 16:52:14,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: finishing write tid 3 to mira0435601-423 2014-07-01 16:52:14,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: finishing write tid 4 to mira0435601-423 2014-07-01 16:52:14,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 212 (ObjNum 1556 snap 0 seq_num 1556) dirty exists 2014-07-01 16:52:14,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: expect (ObjNum 554 snap 0 seq_num 554) 2014-07-01 16:52:14,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 3 to mira0437250-164 2014-07-01 16:52:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 1 to mira0437250-166 2014-07-01 16:52:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 3 to mira0437250-166 2014-07-01 16:52:14,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 1 to mira0437250-167 2014-07-01 16:52:14,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 2 to mira0437250-167 2014-07-01 16:52:14,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 4 to mira0437250-167 2014-07-01 16:52:14,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 1 to mira0437250-169 2014-07-01 16:52:14,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 2 to mira0437250-169 2014-07-01 16:52:14,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 4 to mira0437250-169 2014-07-01 16:52:14,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 1 to mira0437250-173 2014-07-01 16:52:14,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 2 to mira0437250-173 2014-07-01 16:52:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 3 to mira0437250-173 2014-07-01 16:52:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 4 to mira0437250-173 2014-07-01 16:52:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 6 to mira0437250-173 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 1 to mira0437250-174 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 2 to mira0437250-174 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 4 to mira0437250-174 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 1 to mira0437250-172 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 2 to mira0437250-172 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 1 to mira0437250-170 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 2 to mira0437250-170 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 3 to mira0437250-172 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 4 to mira0437250-162 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 4 to mira0437250-170 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 1 to mira0437250-163 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 1 to mira0437250-175 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 5 to mira0437250-172 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 3 to mira0437250-175 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 1 to mira0437250-171 2014-07-01 16:52:15,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 2 to mira0437250-171 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 3 to mira0437250-171 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 4 to mira0437250-171 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 6 to mira0437250-171 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: finishing write tid 4 to mira0437250-175 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 175 v 35 (ObjNum 174 snap 0 seq_num 174) dirty exists 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:175: done (15 left) 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: write initial oid 178 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: seq_num 177 ranges {429555=624631,1818153=535057,2829554=1} 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: writing mira0437250-178 from 429555 to 1054186 tid 1 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: writing mira0437250-178 from 1818153 to 2353210 tid 2 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: writing mira0437250-178 from 2829554 to 2829555 tid 3 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: finishing write tid 7 to mira0437250-171 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 38 (ObjNum 170 snap 0 seq_num 170) dirty exists 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: finishing write tid 6 to mira0437250-172 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 43 (ObjNum 171 snap 0 seq_num 171) dirty exists 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: finishing write tid 4 to mira0437250-166 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 46 (ObjNum 165 snap 0 seq_num 165) dirty exists 2014-07-01 16:52:15,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: finishing write tid 5 to mira0437250-162 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 29 (ObjNum 161 snap 0 seq_num 161) dirty exists 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 2 to mira0437250-163 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:162: done (15 left) 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:166: done (14 left) 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:171: done (13 left) 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:172: done (12 left) 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: write initial oid 179 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: seq_num 178 ranges {525809=728660,2028414=497395} 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0437250-179 from 525809 to 1254469 tid 1 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: writing mira0437250-179 from 2028414 to 2525809 tid 2 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: finishing write tid 4 to mira0437250-164 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 29 (ObjNum 163 snap 0 seq_num 163) dirty exists 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 3 to mira0437250-163 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:164: done (12 left) 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: write initial oid 180 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: seq_num 179 ranges {319214=1} 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: writing mira0437250-180 from 319214 to 319215 tid 1 2014-07-01 16:52:15,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: write initial oid 181 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: seq_num 180 ranges {14443=1} 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: writing mira0437250-181 from 14443 to 14444 tid 1 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: write initial oid 182 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: seq_num 181 ranges {227049=1} 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: writing mira0437250-182 from 227049 to 227050 tid 1 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: write initial oid 183 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: seq_num 182 ranges {569832=400000} 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: writing mira0437250-183 from 569832 to 969832 tid 1 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 5 to mira0437250-163 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 1 to mira0437250-165 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 2 to mira0437250-165 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 3 to mira0437250-165 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 4 to mira0437250-165 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 6 to mira0437250-165 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 1 to mira0437250-177 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 2 to mira0437250-177 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 3 to mira0437250-177 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 5 to mira0437250-177 2014-07-01 16:52:15,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: finishing write tid 6 to mira0437250-163 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 35 (ObjNum 162 snap 0 seq_num 162) dirty exists 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: finishing write tid 5 to mira0437250-167 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 167 v 43 (ObjNum 166 snap 0 seq_num 166) dirty exists 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: finishing write tid 7 to mira0437250-165 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 40 (ObjNum 164 snap 0 seq_num 164) dirty exists 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 1 to mira0437250-168 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 2 to mira0437250-168 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 1 to mira0437250-176 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:163: done (15 left) 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:165: done (14 left) 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:167: done (13 left) 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: write initial oid 184 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: seq_num 183 ranges {514662=527649,1714661=1} 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: writing mira0437250-184 from 514662 to 1042311 tid 1 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: writing mira0437250-184 from 1714661 to 1714662 tid 2 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: write initial oid 185 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: seq_num 184 ranges {610915=499454,1410914=1} 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: writing mira0437250-185 from 610915 to 1110369 tid 1 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: writing mira0437250-185 from 1410914 to 1410915 tid 2 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: write initial oid 186 2014-07-01 16:52:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: seq_num 185 ranges {671073=622682,2036426=633514,2671072=1} 2014-07-01 16:52:15,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: writing mira0437250-186 from 671073 to 1293755 tid 1 2014-07-01 16:52:15,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: writing mira0437250-186 from 2036426 to 2669940 tid 2 2014-07-01 16:52:15,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: writing mira0437250-186 from 2671072 to 2671073 tid 3 2014-07-01 16:52:15,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 3 to mira0437250-176 2014-07-01 16:52:15,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 3 to mira0437250-168 2014-07-01 16:52:15,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 5 to mira0437250-168 2014-07-01 16:52:15,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: finishing write tid 5 to mira0437250-170 2014-07-01 16:52:15,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 36 (ObjNum 169 snap 0 seq_num 169) dirty exists 2014-07-01 16:52:15,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: finishing write tid 6 to mira0437250-177 2014-07-01 16:52:15,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 44 (ObjNum 176 snap 0 seq_num 176) dirty exists 2014-07-01 16:52:15,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 1 to mira0437250-178 2014-07-01 16:52:15,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: finishing write tid 6 to mira0437250-168 2014-07-01 16:52:15,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 28 (ObjNum 167 snap 0 seq_num 167) dirty exists 2014-07-01 16:52:15,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: finishing write tid 5 to mira0437250-169 2014-07-01 16:52:15,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 169 v 49 (ObjNum 168 snap 0 seq_num 168) dirty exists 2014-07-01 16:52:15,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: finishing write tid 4 to mira0437250-176 2014-07-01 16:52:15,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 29 (ObjNum 175 snap 0 seq_num 175) dirty exists 2014-07-01 16:52:15,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: finishing write tid 7 to mira0437250-173 2014-07-01 16:52:15,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 54 (ObjNum 172 snap 0 seq_num 172) dirty exists 2014-07-01 16:52:15,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: finishing write tid 5 to mira0437250-174 2014-07-01 16:52:15,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 32 (ObjNum 173 snap 0 seq_num 173) dirty exists 2014-07-01 16:52:15,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 2 to mira0437250-178 2014-07-01 16:52:15,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:168: done (15 left) 2014-07-01 16:52:15,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:169: done (14 left) 2014-07-01 16:52:15,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:170: done (13 left) 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:173: done (12 left) 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:174: done (11 left) 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:176: done (10 left) 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:177: done (9 left) 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: write initial oid 187 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: seq_num 186 ranges {615903=479159,1882783=703709,3227481=188422} 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: writing mira0437250-187 from 615903 to 1095062 tid 1 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: writing mira0437250-187 from 1882783 to 2586492 tid 2 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: writing mira0437250-187 from 3227481 to 3415903 tid 3 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: write initial oid 188 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: seq_num 187 ranges {711133=602387,1995891=773904,3111132=1} 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0437250-188 from 711133 to 1313520 tid 1 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0437250-188 from 1995891 to 2769795 tid 2 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: writing mira0437250-188 from 3111132 to 3111133 tid 3 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: write initial oid 189 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 3 to mira0437250-178 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 5 to mira0437250-178 2014-07-01 16:52:15,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: finishing write tid 6 to mira0437250-178 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 33 (ObjNum 177 snap 0 seq_num 177) dirty exists 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 1 to mira0437250-179 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 1 to mira0437250-181 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 1 to mira0437250-180 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 3 to mira0437250-181 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: finishing write tid 4 to mira0437250-181 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 31 (ObjNum 180 snap 0 seq_num 180) dirty exists 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 3 to mira0437250-180 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: seq_num 188 ranges {655962=575215,1675974=595524,2947307=624067,3855961=1} 2014-07-01 16:52:15,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0437250-189 from 655962 to 1231177 tid 1 2014-07-01 16:52:15,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0437250-189 from 1675974 to 2271498 tid 2 2014-07-01 16:52:15,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0437250-189 from 2947307 to 3571374 tid 3 2014-07-01 16:52:15,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: writing mira0437250-189 from 3855961 to 3855962 tid 4 2014-07-01 16:52:15,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:178: done (11 left) 2014-07-01 16:52:15,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:181: done (10 left) 2014-07-01 16:52:15,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: write initial oid 190 2014-07-01 16:52:15,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: seq_num 189 ranges {600791=1} 2014-07-01 16:52:15,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: writing mira0437250-190 from 600791 to 600792 tid 1 2014-07-01 16:52:15,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: write initial oid 191 2014-07-01 16:52:15,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: seq_num 190 ranges {411350=400000} 2014-07-01 16:52:15,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: writing mira0437250-191 from 411350 to 811350 tid 1 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: write initial oid 192 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: seq_num 191 ranges {756179=526724,1556178=1} 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: writing mira0437250-192 from 756179 to 1282903 tid 1 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: writing mira0437250-192 from 1556178 to 1556179 tid 2 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: write initial oid 193 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: seq_num 192 ranges {699985=498529,1641111=510352,2299984=1} 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: writing mira0437250-193 from 699985 to 1198514 tid 1 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: writing mira0437250-193 from 1641111 to 2151463 tid 2 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: writing mira0437250-193 from 2299984 to 2299985 tid 3 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: write initial oid 194 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: seq_num 193 ranges {796239=622781,1996238=1} 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0437250-194 from 796239 to 1419020 tid 1 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: writing mira0437250-194 from 1996238 to 1996239 tid 2 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: write initial oid 195 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: seq_num 194 ranges {457421=478233,1434304=650743,2880165=377256} 2014-07-01 16:52:15,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: writing mira0437250-195 from 457421 to 935654 tid 1 2014-07-01 16:52:15,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: writing mira0437250-195 from 1434304 to 2085047 tid 2 2014-07-01 16:52:15,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: writing mira0437250-195 from 2880165 to 3257421 tid 3 2014-07-01 16:52:15,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: finishing write tid 4 to mira0437250-180 2014-07-01 16:52:15,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 35 (ObjNum 179 snap 0 seq_num 179) dirty exists 2014-07-01 16:52:15,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:180: done (15 left) 2014-07-01 16:52:15,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: write initial oid 196 2014-07-01 16:52:15,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: seq_num 195 ranges {552650=601462,1815188=472362,2831790=120860} 2014-07-01 16:52:15,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0437250-196 from 552650 to 1154112 tid 1 2014-07-01 16:52:15,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0437250-196 from 1815188 to 2287550 tid 2 2014-07-01 16:52:15,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: writing mira0437250-196 from 2831790 to 2952650 tid 3 2014-07-01 16:52:15,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 2 to mira0437250-179 2014-07-01 16:52:15,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 4 to mira0437250-179 2014-07-01 16:52:15,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: finishing write tid 5 to mira0437250-179 2014-07-01 16:52:15,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 179 v 35 (ObjNum 178 snap 0 seq_num 178) dirty exists 2014-07-01 16:52:15,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 1 to mira0437250-182 2014-07-01 16:52:15,518.518 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:52:15,535.535 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:52:15,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 3 to mira0437250-182 2014-07-01 16:52:15,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: finishing write tid 4 to mira0437250-182 2014-07-01 16:52:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 45 (ObjNum 181 snap 0 seq_num 181) dirty exists 2014-07-01 16:52:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:179: done (15 left) 2014-07-01 16:52:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:182: done (14 left) 2014-07-01 16:52:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: write initial oid 197 2014-07-01 16:52:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: seq_num 196 ranges {497480=573266,1777896=693981,3185463=512017} 2014-07-01 16:52:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0437250-197 from 497480 to 1070746 tid 1 2014-07-01 16:52:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0437250-197 from 1777896 to 2471877 tid 2 2014-07-01 16:52:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: writing mira0437250-197 from 3185463 to 3697480 tid 3 2014-07-01 16:52:15,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: write initial oid 198 2014-07-01 16:52:15,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: seq_num 197 ranges {441285=1} 2014-07-01 16:52:15,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: writing mira0437250-198 from 441285 to 441286 tid 1 2014-07-01 16:52:15,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:15,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 1 to mira0437250-183 2014-07-01 16:52:15,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 3 to mira0437250-183 2014-07-01 16:52:15,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: finishing write tid 4 to mira0437250-183 2014-07-01 16:52:15,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 28 (ObjNum 182 snap 0 seq_num 182) dirty exists 2014-07-01 16:52:15,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 1 to mira0437250-184 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 2 to mira0437250-184 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 4 to mira0437250-184 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: finishing write tid 5 to mira0437250-184 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 27 (ObjNum 183 snap 0 seq_num 183) dirty exists 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 1 to mira0437250-185 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 2 to mira0437250-185 2014-07-01 16:52:15,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 4 to mira0437250-185 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: finishing write tid 5 to mira0437250-185 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 48 (ObjNum 184 snap 0 seq_num 184) dirty exists 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 1 to mira0437250-186 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 2 to mira0437250-186 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 3 to mira0437250-186 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 5 to mira0437250-186 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: finishing write tid 6 to mira0437250-186 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 39 (ObjNum 185 snap 0 seq_num 185) dirty exists 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 1 to mira0437250-187 2014-07-01 16:52:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 2 to mira0437250-187 2014-07-01 16:52:15,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 3 to mira0437250-187 2014-07-01 16:52:15,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 5 to mira0437250-187 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: finishing write tid 6 to mira0437250-187 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 43 (ObjNum 186 snap 0 seq_num 186) dirty exists 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 1 to mira0437250-188 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 2 to mira0437250-188 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 3 to mira0437250-188 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 5 to mira0437250-188 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: finishing write tid 6 to mira0437250-188 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 31 (ObjNum 187 snap 0 seq_num 187) dirty exists 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 1 to mira0437250-189 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 2 to mira0437250-189 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 3 to mira0437250-189 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 1 to mira0437250-190 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 4 to mira0437250-189 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 3 to mira0437250-190 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 6 to mira0437250-189 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: finishing write tid 4 to mira0437250-190 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 30 (ObjNum 189 snap 0 seq_num 189) dirty exists 2014-07-01 16:52:15,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: finishing write tid 7 to mira0437250-189 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 40 (ObjNum 188 snap 0 seq_num 188) dirty exists 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 1 to mira0437250-191 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 3 to mira0437250-191 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: finishing write tid 4 to mira0437250-191 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 30 (ObjNum 190 snap 0 seq_num 190) dirty exists 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 1 to mira0437250-192 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 2 to mira0437250-192 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 4 to mira0437250-192 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: finishing write tid 5 to mira0437250-192 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 34 (ObjNum 191 snap 0 seq_num 191) dirty exists 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 1 to mira0437250-193 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 2 to mira0437250-193 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 3 to mira0437250-193 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 5 to mira0437250-193 2014-07-01 16:52:15,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: finishing write tid 6 to mira0437250-193 2014-07-01 16:52:15,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 38 (ObjNum 192 snap 0 seq_num 192) dirty exists 2014-07-01 16:52:15,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 1 to mira0437250-194 2014-07-01 16:52:15,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 2 to mira0437250-194 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 4 to mira0437250-194 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: finishing write tid 5 to mira0437250-194 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 46 (ObjNum 193 snap 0 seq_num 193) dirty exists 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 1 to mira0437250-195 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 2 to mira0437250-195 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 3 to mira0437250-195 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 5 to mira0437250-195 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: finishing write tid 6 to mira0437250-195 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 42 (ObjNum 194 snap 0 seq_num 194) dirty exists 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 1 to mira0437250-196 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 2 to mira0437250-196 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 3 to mira0437250-196 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 5 to mira0437250-196 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: finishing write tid 6 to mira0437250-196 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 39 (ObjNum 195 snap 0 seq_num 195) dirty exists 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 1 to mira0437250-197 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 2 to mira0437250-197 2014-07-01 16:52:15,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:183: done (15 left) 2014-07-01 16:52:15,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:184: done (14 left) 2014-07-01 16:52:15,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:185: done (13 left) 2014-07-01 16:52:15,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:186: done (12 left) 2014-07-01 16:52:15,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:187: done (11 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:188: done (10 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:189: done (9 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:190: done (8 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:191: done (7 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:192: done (6 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:193: done (5 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:194: done (4 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:195: done (3 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:196: done (2 left) 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: write initial oid 199 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: seq_num 198 ranges {136515=1} 2014-07-01 16:52:15,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: writing mira0437250-199 from 136515 to 136516 tid 1 2014-07-01 16:52:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: write initial oid 200 2014-07-01 16:52:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: seq_num 199 ranges {597697=676199,1397696=1} 2014-07-01 16:52:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: writing mira0437250-200 from 597697 to 1273896 tid 1 2014-07-01 16:52:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: writing mira0437250-200 from 1397696 to 1397697 tid 2 2014-07-01 16:52:15,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: write initial oid 201 2014-07-01 16:52:15,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: seq_num 200 ranges {541503=497603,1460408=609834,2141502=1} 2014-07-01 16:52:15,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: writing mira0437250-201 from 541503 to 1039106 tid 1 2014-07-01 16:52:15,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: writing mira0437250-201 from 1460408 to 2070242 tid 2 2014-07-01 16:52:15,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: writing mira0437250-201 from 2141502 to 2141503 tid 3 2014-07-01 16:52:15,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: write initial oid 202 2014-07-01 16:52:15,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: seq_num 201 ranges {637757=620832,1725965=111792} 2014-07-01 16:52:15,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0437250-202 from 637757 to 1258589 tid 1 2014-07-01 16:52:15,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: writing mira0437250-202 from 1725965 to 1837757 tid 2 2014-07-01 16:52:15,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: write initial oid 203 2014-07-01 16:52:15,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: seq_num 202 ranges {581563=476284,1570273=750224,2581562=1} 2014-07-01 16:52:15,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0437250-203 from 581563 to 1057847 tid 1 2014-07-01 16:52:15,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0437250-203 from 1570273 to 2320497 tid 2 2014-07-01 16:52:15,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: writing mira0437250-203 from 2581562 to 2581563 tid 3 2014-07-01 16:52:15,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: write initial oid 204 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: seq_num 203 ranges {642744=599512,1765684=571843,3035895=634469,3842743=1} 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: writing mira0437250-204 from 642744 to 1242256 tid 1 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: writing mira0437250-204 from 1765684 to 2337527 tid 2 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: writing mira0437250-204 from 3035895 to 3670364 tid 3 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: writing mira0437250-204 from 3842743 to 3842744 tid 4 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: write initial oid 205 2014-07-01 16:52:15,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: seq_num 204 ranges {738998=572341,1880841=793463,3274242=264756} 2014-07-01 16:52:15,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0437250-205 from 738998 to 1311339 tid 1 2014-07-01 16:52:15,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0437250-205 from 1880841 to 2674304 tid 2 2014-07-01 16:52:15,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: writing mira0437250-205 from 3274242 to 3538998 tid 3 2014-07-01 16:52:15,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: write initial oid 206 2014-07-01 16:52:15,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: seq_num 205 ranges {434228=695569,1861725=712234,3190795=43433} 2014-07-01 16:52:15,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: writing mira0437250-206 from 434228 to 1129797 tid 1 2014-07-01 16:52:15,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: writing mira0437250-206 from 1861725 to 2573959 tid 2 2014-07-01 16:52:15,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: writing mira0437250-206 from 3190795 to 3234228 tid 3 2014-07-01 16:52:15,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: write initial oid 207 2014-07-01 16:52:15,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: seq_num 206 ranges {626610=400000} 2014-07-01 16:52:15,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: writing mira0437250-207 from 626610 to 1026610 tid 1 2014-07-01 16:52:15,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: write initial oid 208 2014-07-01 16:52:15,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: seq_num 207 ranges {721838=1} 2014-07-01 16:52:15,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: writing mira0437250-208 from 721838 to 721839 tid 1 2014-07-01 16:52:15,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: write initial oid 209 2014-07-01 16:52:15,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: seq_num 208 ranges {783021=647078,1865177=117844} 2014-07-01 16:52:15,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: writing mira0437250-209 from 783021 to 1430099 tid 1 2014-07-01 16:52:15,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: writing mira0437250-209 from 1865177 to 1983021 tid 2 2014-07-01 16:52:15,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: write initial oid 210 2014-07-01 16:52:15,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: seq_num 209 ranges {727851=770306,2226861=500990} 2014-07-01 16:52:15,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: writing mira0437250-210 from 727851 to 1498157 tid 1 2014-07-01 16:52:15,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: writing mira0437250-210 from 2226861 to 2727851 tid 2 2014-07-01 16:52:15,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: write initial oid 211 2014-07-01 16:52:15,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: seq_num 210 ranges {423080=474335,1388546=449706,2423079=1} 2014-07-01 16:52:15,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0437250-211 from 423080 to 897415 tid 1 2014-07-01 16:52:16,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0437250-211 from 1388546 to 1838252 tid 2 2014-07-01 16:52:16,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: writing mira0437250-211 from 2423079 to 2423080 tid 3 2014-07-01 16:52:16,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: write initial oid 212 2014-07-01 16:52:16,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: seq_num 211 ranges {767910=598587,1903702=519901,3008324=159586} 2014-07-01 16:52:16,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0437250-212 from 767910 to 1366497 tid 1 2014-07-01 16:52:16,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0437250-212 from 1903702 to 2423603 tid 2 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: writing mira0437250-212 from 3008324 to 3167910 tid 3 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 3 to mira0437250-197 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 1 to mira0437250-199 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 5 to mira0437250-197 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 3 to mira0437250-199 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: finishing write tid 6 to mira0437250-197 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 197 v 34 (ObjNum 196 snap 0 seq_num 196) dirty exists 2014-07-01 16:52:16,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: finishing write tid 4 to mira0437250-199 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 40 (ObjNum 198 snap 0 seq_num 198) dirty exists 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 1 to mira0437250-198 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 3 to mira0437250-198 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: finishing write tid 4 to mira0437250-198 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 46 (ObjNum 197 snap 0 seq_num 197) dirty exists 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 1 to mira0437250-200 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 2 to mira0437250-200 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 4 to mira0437250-200 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: finishing write tid 5 to mira0437250-200 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 33 (ObjNum 199 snap 0 seq_num 199) dirty exists 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 1 to mira0437250-201 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 2 to mira0437250-201 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 3 to mira0437250-201 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 5 to mira0437250-201 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: finishing write tid 6 to mira0437250-201 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 201 v 50 (ObjNum 200 snap 0 seq_num 200) dirty exists 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 1 to mira0437250-202 2014-07-01 16:52:16,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:197: done (15 left) 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:198: done (14 left) 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:199: done (13 left) 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:200: done (12 left) 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:201: done (11 left) 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: write initial oid 213 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: seq_num 212 ranges {428067=1} 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: writing mira0437250-213 from 428067 to 428068 tid 1 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: write initial oid 214 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: seq_num 213 ranges {124321=1} 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: writing mira0437250-214 from 124321 to 124322 tid 1 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: write initial oid 215 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: seq_num 214 ranges {468127=400000} 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: writing mira0437250-215 from 468127 to 868127 tid 1 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: write initial oid 216 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: seq_num 215 ranges {411933=673324,1611932=1} 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: writing mira0437250-216 from 411933 to 1085257 tid 1 2014-07-01 16:52:16,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: writing mira0437250-216 from 1611932 to 1611933 tid 2 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: write initial oid 217 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: seq_num 216 ranges {508187=645129,1308186=1} 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: writing mira0437250-217 from 508187 to 1153316 tid 1 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: writing mira0437250-217 from 1308186 to 1308187 tid 2 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 2 to mira0437250-202 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 4 to mira0437250-202 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: finishing write tid 5 to mira0437250-202 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 43 (ObjNum 201 snap 0 seq_num 201) dirty exists 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 1 to mira0437250-203 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 2 to mira0437250-203 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 3 to mira0437250-203 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 5 to mira0437250-203 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: finishing write tid 6 to mira0437250-203 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 37 (ObjNum 202 snap 0 seq_num 202) dirty exists 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 1 to mira0437250-204 2014-07-01 16:52:16,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 2 to mira0437250-204 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 3 to mira0437250-204 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 4 to mira0437250-204 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 6 to mira0437250-204 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: finishing write tid 7 to mira0437250-204 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 43 (ObjNum 203 snap 0 seq_num 203) dirty exists 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 1 to mira0437250-205 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:202: done (15 left) 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:203: done (14 left) 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:204: done (13 left) 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: write initial oid 218 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: seq_num 217 ranges {568345=769381,2080207=477968,2568344=1} 2014-07-01 16:52:16,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: writing mira0437250-218 from 568345 to 1337726 tid 1 2014-07-01 16:52:16,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: writing mira0437250-218 from 2080207 to 2558175 tid 2 2014-07-01 16:52:16,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: writing mira0437250-218 from 2568344 to 2568345 tid 3 2014-07-01 16:52:16,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: write initial oid 219 2014-07-01 16:52:16,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: seq_num 218 ranges {664598=625858,2043939=220659} 2014-07-01 16:52:16,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: writing mira0437250-219 from 664598 to 1290456 tid 1 2014-07-01 16:52:16,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: writing mira0437250-219 from 2043939 to 2264598 tid 2 2014-07-01 16:52:16,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: write initial oid 220 2014-07-01 16:52:16,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: seq_num 219 ranges {609428=749086,2006647=618358,3009427=1} 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0437250-220 from 609428 to 1358514 tid 1 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0437250-220 from 2006647 to 2625005 tid 2 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: writing mira0437250-220 from 3009427 to 3009428 tid 3 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 2 to mira0437250-205 2014-07-01 16:52:16,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 3 to mira0437250-205 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 5 to mira0437250-205 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: finishing write tid 6 to mira0437250-205 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 205 v 47 (ObjNum 204 snap 0 seq_num 204) dirty exists 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 1 to mira0437250-206 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 2 to mira0437250-206 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 3 to mira0437250-206 2014-07-01 16:52:16,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:205: done (15 left) 2014-07-01 16:52:16,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: write initial oid 221 2014-07-01 16:52:16,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: seq_num 220 ranges {553234=720890,1684683=439978,2765080=513556,3753233=1} 2014-07-01 16:52:16,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0437250-221 from 553234 to 1274124 tid 1 2014-07-01 16:52:16,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0437250-221 from 1684683 to 2124661 tid 2 2014-07-01 16:52:16,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0437250-221 from 2765080 to 3278636 tid 3 2014-07-01 16:52:16,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: writing mira0437250-221 from 3753233 to 3753234 tid 4 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 5 to mira0437250-206 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: finishing write tid 6 to mira0437250-206 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 47 (ObjNum 205 snap 0 seq_num 205) dirty exists 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 1 to mira0437250-208 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 3 to mira0437250-208 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 1 to mira0437250-207 2014-07-01 16:52:16,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:206: done (15 left) 2014-07-01 16:52:16,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: write initial oid 222 2014-07-01 16:52:16,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: seq_num 221 ranges {765840=692695,2163744=510173,3331234=400347,3965839=1} 2014-07-01 16:52:16,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0437250-222 from 765840 to 1458535 tid 1 2014-07-01 16:52:16,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0437250-222 from 2163744 to 2673917 tid 2 2014-07-01 16:52:16,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0437250-222 from 3331234 to 3731581 tid 3 2014-07-01 16:52:16,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: writing mira0437250-222 from 3965839 to 3965840 tid 4 2014-07-01 16:52:16,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 3 to mira0437250-207 2014-07-01 16:52:16,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: finishing write tid 4 to mira0437250-208 2014-07-01 16:52:16,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 39 (ObjNum 207 snap 0 seq_num 207) dirty exists 2014-07-01 16:52:16,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: finishing write tid 4 to mira0437250-207 2014-07-01 16:52:16,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 207 v 49 (ObjNum 206 snap 0 seq_num 206) dirty exists 2014-07-01 16:52:16,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 1 to mira0437250-209 2014-07-01 16:52:16,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 2 to mira0437250-209 2014-07-01 16:52:16,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 4 to mira0437250-209 2014-07-01 16:52:16,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:207: done (15 left) 2014-07-01 16:52:16,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:208: done (14 left) 2014-07-01 16:52:16,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: write initial oid 223 2014-07-01 16:52:16,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: seq_num 222 ranges {708620=1} 2014-07-01 16:52:16,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: writing mira0437250-223 from 708620 to 708621 tid 1 2014-07-01 16:52:16,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: write initial oid 224 2014-07-01 16:52:16,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: seq_num 223 ranges {653451=672399,1453450=1} 2014-07-01 16:52:16,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: writing mira0437250-224 from 653451 to 1325850 tid 1 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: writing mira0437250-224 from 1453450 to 1453451 tid 2 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: finishing write tid 5 to mira0437250-209 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 209 v 52 (ObjNum 208 snap 0 seq_num 208) dirty exists 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 1 to mira0437250-210 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 2 to mira0437250-210 2014-07-01 16:52:16,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:209: done (15 left) 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: write initial oid 225 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: seq_num 224 ranges {749705=400000} 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: writing mira0437250-225 from 749705 to 1149705 tid 1 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 4 to mira0437250-210 2014-07-01 16:52:16,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: finishing write tid 5 to mira0437250-210 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 210 v 46 (ObjNum 209 snap 0 seq_num 209) dirty exists 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 1 to mira0437250-211 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 2 to mira0437250-211 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 3 to mira0437250-211 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 5 to mira0437250-211 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: finishing write tid 6 to mira0437250-211 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 50 (ObjNum 210 snap 0 seq_num 210) dirty exists 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 1 to mira0437250-212 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 2 to mira0437250-212 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 1 to mira0437250-213 2014-07-01 16:52:16,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 3 to mira0437250-212 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 3 to mira0437250-213 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 5 to mira0437250-212 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: finishing write tid 6 to mira0437250-212 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 43 (ObjNum 211 snap 0 seq_num 211) dirty exists 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: finishing write tid 4 to mira0437250-213 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 50 (ObjNum 212 snap 0 seq_num 212) dirty exists 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 1 to mira0437250-214 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 3 to mira0437250-214 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: finishing write tid 4 to mira0437250-214 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 48 (ObjNum 213 snap 0 seq_num 213) dirty exists 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 1 to mira0437250-215 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:210: done (15 left) 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:211: done (14 left) 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:212: done (13 left) 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:213: done (12 left) 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:214: done (11 left) 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: write initial oid 226 2014-07-01 16:52:16,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: seq_num 225 ranges {693511=768456,1893510=1} 2014-07-01 16:52:16,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0437250-226 from 693511 to 1461967 tid 1 2014-07-01 16:52:16,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: writing mira0437250-226 from 1893510 to 1893511 tid 2 2014-07-01 16:52:16,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: write initial oid 227 2014-07-01 16:52:16,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: seq_num 226 ranges {754692=740260,2110787=647645,3154691=1} 2014-07-01 16:52:16,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: writing mira0437250-227 from 754692 to 1494952 tid 1 2014-07-01 16:52:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: writing mira0437250-227 from 2110787 to 2758432 tid 2 2014-07-01 16:52:16,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: writing mira0437250-227 from 3154691 to 3154692 tid 3 2014-07-01 16:52:16,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: write initial oid 228 2014-07-01 16:52:16,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: seq_num 227 ranges {450946=747137,1708569=717840,2850945=1} 2014-07-01 16:52:16,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0437250-228 from 450946 to 1198083 tid 1 2014-07-01 16:52:16,546.546 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:52:16,560.560 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:52:16,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0437250-228 from 1708569 to 2426409 tid 2 2014-07-01 16:52:16,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: writing mira0437250-228 from 2850945 to 2850946 tid 3 2014-07-01 16:52:16,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: write initial oid 229 2014-07-01 16:52:16,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: seq_num 228 ranges {794752=718941,2186605=539459,3520612=74140} 2014-07-01 16:52:16,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0437250-229 from 794752 to 1513693 tid 1 2014-07-01 16:52:16,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0437250-229 from 2186605 to 2726064 tid 2 2014-07-01 16:52:16,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: writing mira0437250-229 from 3520612 to 3594752 tid 3 2014-07-01 16:52:16,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: write initial oid 230 2014-07-01 16:52:16,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: seq_num 229 ranges {339580=1} 2014-07-01 16:52:16,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: writing mira0437250-230 from 339580 to 339581 tid 1 2014-07-01 16:52:16,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 3 to mira0437250-215 2014-07-01 16:52:16,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: finishing write tid 4 to mira0437250-215 2014-07-01 16:52:16,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 42 (ObjNum 214 snap 0 seq_num 214) dirty exists 2014-07-01 16:52:16,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 1 to mira0437250-216 2014-07-01 16:52:16,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 2 to mira0437250-216 2014-07-01 16:52:16,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 4 to mira0437250-216 2014-07-01 16:52:16,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: finishing write tid 5 to mira0437250-216 2014-07-01 16:52:16,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 216 v 57 (ObjNum 215 snap 0 seq_num 215) dirty exists 2014-07-01 16:52:16,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 1 to mira0437250-217 2014-07-01 16:52:16,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:215: done (15 left) 2014-07-01 16:52:16,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:216: done (14 left) 2014-07-01 16:52:16,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: write initial oid 231 2014-07-01 16:52:16,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: seq_num 230 ranges {551162=1} 2014-07-01 16:52:16,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: writing mira0437250-231 from 551162 to 551163 tid 1 2014-07-01 16:52:16,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: write initial oid 232 2014-07-01 16:52:16,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: seq_num 231 ranges {246392=1} 2014-07-01 16:52:16,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: writing mira0437250-232 from 246392 to 246393 tid 1 2014-07-01 16:52:16,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 2 to mira0437250-217 2014-07-01 16:52:16,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 4 to mira0437250-217 2014-07-01 16:52:16,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: finishing write tid 5 to mira0437250-217 2014-07-01 16:52:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 38 (ObjNum 216 snap 0 seq_num 216) dirty exists 2014-07-01 16:52:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 1 to mira0437250-218 2014-07-01 16:52:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 2 to mira0437250-218 2014-07-01 16:52:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:217: done (15 left) 2014-07-01 16:52:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: write initial oid 233 2014-07-01 16:52:16,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: seq_num 232 ranges {439799=643278,1505213=419216,2039798=1} 2014-07-01 16:52:16,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: writing mira0437250-233 from 439799 to 1083077 tid 1 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: writing mira0437250-233 from 1505213 to 1924429 tid 2 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: writing mira0437250-233 from 2039798 to 2039799 tid 3 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 3 to mira0437250-218 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 5 to mira0437250-218 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: finishing write tid 6 to mira0437250-218 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 42 (ObjNum 217 snap 0 seq_num 217) dirty exists 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 1 to mira0437250-219 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:218: done (15 left) 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: write initial oid 234 2014-07-01 16:52:16,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: seq_num 233 ranges {535028=766507,1735027=1} 2014-07-01 16:52:16,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0437250-234 from 535028 to 1301535 tid 1 2014-07-01 16:52:16,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: writing mira0437250-234 from 1735027 to 1735028 tid 2 2014-07-01 16:52:16,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 2 to mira0437250-219 2014-07-01 16:52:16,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 4 to mira0437250-219 2014-07-01 16:52:16,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: finishing write tid 5 to mira0437250-219 2014-07-01 16:52:16,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 51 (ObjNum 218 snap 0 seq_num 218) dirty exists 2014-07-01 16:52:16,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 1 to mira0437250-220 2014-07-01 16:52:16,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 2 to mira0437250-220 2014-07-01 16:52:16,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:219: done (15 left) 2014-07-01 16:52:16,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: write initial oid 235 2014-07-01 16:52:16,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: seq_num 234 ranges {631282=738311,1431281=1} 2014-07-01 16:52:16,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: writing mira0437250-235 from 631282 to 1369593 tid 1 2014-07-01 16:52:16,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: writing mira0437250-235 from 1431281 to 1431282 tid 2 2014-07-01 16:52:16,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 3 to mira0437250-220 2014-07-01 16:52:16,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 5 to mira0437250-220 2014-07-01 16:52:16,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: finishing write tid 6 to mira0437250-220 2014-07-01 16:52:16,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 46 (ObjNum 219 snap 0 seq_num 219) dirty exists 2014-07-01 16:52:16,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 1 to mira0437250-221 2014-07-01 16:52:16,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 2 to mira0437250-221 2014-07-01 16:52:16,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 3 to mira0437250-221 2014-07-01 16:52:16,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:220: done (15 left) 2014-07-01 16:52:16,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: write initial oid 236 2014-07-01 16:52:16,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: seq_num 235 ranges {540016=745187,1925817=416298,2853669=675382,3740015=1} 2014-07-01 16:52:16,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: writing mira0437250-236 from 540016 to 1285203 tid 1 2014-07-01 16:52:16,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: writing mira0437250-236 from 1925817 to 2342115 tid 2 2014-07-01 16:52:16,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: writing mira0437250-236 from 2853669 to 3529051 tid 3 2014-07-01 16:52:16,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: writing mira0437250-236 from 3740015 to 3740016 tid 4 2014-07-01 16:52:16,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 4 to mira0437250-221 2014-07-01 16:52:16,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 6 to mira0437250-221 2014-07-01 16:52:16,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: finishing write tid 7 to mira0437250-221 2014-07-01 16:52:16,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 62 (ObjNum 220 snap 0 seq_num 220) dirty exists 2014-07-01 16:52:16,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 1 to mira0437250-222 2014-07-01 16:52:16,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 2 to mira0437250-222 2014-07-01 16:52:16,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 3 to mira0437250-222 2014-07-01 16:52:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 4 to mira0437250-222 2014-07-01 16:52:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:221: done (15 left) 2014-07-01 16:52:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: write initial oid 237 2014-07-01 16:52:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: seq_num 236 ranges {636270=718016,2040974=486493,3207343=228927} 2014-07-01 16:52:16,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0437250-237 from 636270 to 1354286 tid 1 2014-07-01 16:52:16,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0437250-237 from 2040974 to 2527467 tid 2 2014-07-01 16:52:16,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: writing mira0437250-237 from 3207343 to 3436270 tid 3 2014-07-01 16:52:16,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 6 to mira0437250-222 2014-07-01 16:52:16,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 1 to mira0437250-223 2014-07-01 16:52:16,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 3 to mira0437250-223 2014-07-01 16:52:16,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: finishing write tid 4 to mira0437250-223 2014-07-01 16:52:16,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 52 (ObjNum 222 snap 0 seq_num 222) dirty exists 2014-07-01 16:52:16,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: finishing write tid 7 to mira0437250-222 2014-07-01 16:52:16,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 56 (ObjNum 221 snap 0 seq_num 221) dirty exists 2014-07-01 16:52:16,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 1 to mira0437250-224 2014-07-01 16:52:16,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 2 to mira0437250-224 2014-07-01 16:52:16,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 4 to mira0437250-224 2014-07-01 16:52:16,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: finishing write tid 5 to mira0437250-224 2014-07-01 16:52:16,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 49 (ObjNum 223 snap 0 seq_num 223) dirty exists 2014-07-01 16:52:16,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 1 to mira0437250-225 2014-07-01 16:52:16,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:222: done (15 left) 2014-07-01 16:52:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:223: done (14 left) 2014-07-01 16:52:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:224: done (13 left) 2014-07-01 16:52:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: write initial oid 238 2014-07-01 16:52:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: seq_num 237 ranges {732523=441244,1755105=708112,3009591=122932} 2014-07-01 16:52:16,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: writing mira0437250-238 from 732523 to 1173767 tid 1 2014-07-01 16:52:16,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: writing mira0437250-238 from 1755105 to 2463217 tid 2 2014-07-01 16:52:16,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: writing mira0437250-238 from 3009591 to 3132523 tid 3 2014-07-01 16:52:16,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: write initial oid 239 2014-07-01 16:52:16,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: seq_num 238 ranges {524905=400000} 2014-07-01 16:52:16,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: writing mira0437250-239 from 524905 to 924905 tid 1 2014-07-01 16:52:16,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: write initial oid 240 2014-07-01 16:52:16,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: seq_num 239 ranges {736487=400000} 2014-07-01 16:52:16,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: expect (ObjNum 117 snap 0 seq_num 117) 2014-07-01 16:52:16,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: writing mira0437250-240 from 736487 to 1136487 tid 1 2014-07-01 16:52:16,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:16,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 3 to mira0437250-225 2014-07-01 16:52:16,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: finishing write tid 4 to mira0437250-225 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 45 (ObjNum 224 snap 0 seq_num 224) dirty exists 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 1 to mira0437250-226 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 2 to mira0437250-226 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 4 to mira0437250-226 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: finishing write tid 5 to mira0437250-226 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 48 (ObjNum 225 snap 0 seq_num 225) dirty exists 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 1 to mira0437250-227 2014-07-01 16:52:16,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:225: done (15 left) 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:226: done (14 left) 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: write initial oid 241 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: seq_num 240 ranges {431717=400000} 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: writing mira0437250-241 from 431717 to 831717 tid 1 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: write initial oid 242 2014-07-01 16:52:16,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: seq_num 241 ranges {625122=515981,1586993=740317,2625121=1} 2014-07-01 16:52:16,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: writing mira0437250-242 from 625122 to 1141103 tid 1 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: writing mira0437250-242 from 1586993 to 2327310 tid 2 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: writing mira0437250-242 from 2625121 to 2625122 tid 3 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 2 to mira0437250-227 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 3 to mira0437250-227 2014-07-01 16:52:17,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 5 to mira0437250-227 2014-07-01 16:52:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: finishing write tid 6 to mira0437250-227 2014-07-01 16:52:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 56 (ObjNum 226 snap 0 seq_num 226) dirty exists 2014-07-01 16:52:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 1 to mira0437250-228 2014-07-01 16:52:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 2 to mira0437250-228 2014-07-01 16:52:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 3 to mira0437250-228 2014-07-01 16:52:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 5 to mira0437250-228 2014-07-01 16:52:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: finishing write tid 6 to mira0437250-228 2014-07-01 16:52:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 60 (ObjNum 227 snap 0 seq_num 227) dirty exists 2014-07-01 16:52:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 1 to mira0437250-229 2014-07-01 16:52:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:227: done (15 left) 2014-07-01 16:52:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:228: done (14 left) 2014-07-01 16:52:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: write initial oid 243 2014-07-01 16:52:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: seq_num 242 ranges {721376=487786,1701126=561936,2321375=1} 2014-07-01 16:52:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0437250-243 from 721376 to 1209162 tid 1 2014-07-01 16:52:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0437250-243 from 1701126 to 2263062 tid 2 2014-07-01 16:52:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: writing mira0437250-243 from 2321375 to 2321376 tid 3 2014-07-01 16:52:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: write initial oid 244 2014-07-01 16:52:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: seq_num 243 ranges {416606=744262,1663834=352772} 2014-07-01 16:52:17,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: writing mira0437250-244 from 416606 to 1160868 tid 1 2014-07-01 16:52:17,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: writing mira0437250-244 from 1663834 to 2016606 tid 2 2014-07-01 16:52:17,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 2 to mira0437250-229 2014-07-01 16:52:17,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 1 to mira0437250-230 2014-07-01 16:52:17,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 3 to mira0437250-229 2014-07-01 16:52:17,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 5 to mira0437250-229 2014-07-01 16:52:17,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: finishing write tid 6 to mira0437250-229 2014-07-01 16:52:17,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 60 (ObjNum 228 snap 0 seq_num 228) dirty exists 2014-07-01 16:52:17,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 3 to mira0437250-230 2014-07-01 16:52:17,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 1 to mira0437250-232 2014-07-01 16:52:17,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:229: done (15 left) 2014-07-01 16:52:17,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: write initial oid 245 2014-07-01 16:52:17,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: seq_num 244 ranges {477788=716066,1742895=585974,2895098=382690} 2014-07-01 16:52:17,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0437250-245 from 477788 to 1193854 tid 1 2014-07-01 16:52:17,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0437250-245 from 1742895 to 2328869 tid 2 2014-07-01 16:52:17,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: writing mira0437250-245 from 2895098 to 3277788 tid 3 2014-07-01 16:52:17,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 1 to mira0437250-231 2014-07-01 16:52:17,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: finishing write tid 4 to mira0437250-230 2014-07-01 16:52:17,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 44 (ObjNum 229 snap 0 seq_num 229) dirty exists 2014-07-01 16:52:17,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 3 to mira0437250-232 2014-07-01 16:52:17,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: finishing write tid 4 to mira0437250-232 2014-07-01 16:52:17,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 52 (ObjNum 231 snap 0 seq_num 231) dirty exists 2014-07-01 16:52:17,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 3 to mira0437250-231 2014-07-01 16:52:17,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: finishing write tid 4 to mira0437250-231 2014-07-01 16:52:17,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 44 (ObjNum 230 snap 0 seq_num 230) dirty exists 2014-07-01 16:52:17,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 1 to mira0437250-233 2014-07-01 16:52:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:230: done (15 left) 2014-07-01 16:52:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:231: done (14 left) 2014-07-01 16:52:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:232: done (13 left) 2014-07-01 16:52:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: write initial oid 246 2014-07-01 16:52:17,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: seq_num 245 ranges {21592=1} 2014-07-01 16:52:17,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: writing mira0437250-246 from 21592 to 21593 tid 1 2014-07-01 16:52:17,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: write initial oid 247 2014-07-01 16:52:17,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: seq_num 246 ranges {517847=413147,1537111=726365,2713101=644542,3717846=1} 2014-07-01 16:52:17,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: writing mira0437250-247 from 517847 to 930994 tid 1 2014-07-01 16:52:17,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: writing mira0437250-247 from 1537111 to 2263476 tid 2 2014-07-01 16:52:17,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: writing mira0437250-247 from 2713101 to 3357643 tid 3 2014-07-01 16:52:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: writing mira0437250-247 from 3717846 to 3717847 tid 4 2014-07-01 16:52:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: write initial oid 248 2014-07-01 16:52:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: seq_num 247 ranges {461652=1} 2014-07-01 16:52:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: writing mira0437250-248 from 461652 to 461653 tid 1 2014-07-01 16:52:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 2 to mira0437250-233 2014-07-01 16:52:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 3 to mira0437250-233 2014-07-01 16:52:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 5 to mira0437250-233 2014-07-01 16:52:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: finishing write tid 6 to mira0437250-233 2014-07-01 16:52:17,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 47 (ObjNum 232 snap 0 seq_num 232) dirty exists 2014-07-01 16:52:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 1 to mira0437250-234 2014-07-01 16:52:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 2 to mira0437250-234 2014-07-01 16:52:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 4 to mira0437250-234 2014-07-01 16:52:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: finishing write tid 5 to mira0437250-234 2014-07-01 16:52:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 37 (ObjNum 233 snap 0 seq_num 233) dirty exists 2014-07-01 16:52:17,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 1 to mira0437250-235 2014-07-01 16:52:17,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 2 to mira0437250-235 2014-07-01 16:52:17,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 4 to mira0437250-235 2014-07-01 16:52:17,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:233: done (15 left) 2014-07-01 16:52:17,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:234: done (14 left) 2014-07-01 16:52:17,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: write initial oid 249 2014-07-01 16:52:17,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: seq_num 248 ranges {521810=791828,1721809=1} 2014-07-01 16:52:17,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: writing mira0437250-249 from 521810 to 1313638 tid 1 2014-07-01 16:52:17,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: writing mira0437250-249 from 1721809 to 1721810 tid 2 2014-07-01 16:52:17,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: write initial oid 250 2014-07-01 16:52:17,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: seq_num 249 ranges {618064=515056,1418063=1} 2014-07-01 16:52:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0437250-250 from 618064 to 1133120 tid 1 2014-07-01 16:52:17,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: writing mira0437250-250 from 1418063 to 1418064 tid 2 2014-07-01 16:52:17,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: finishing write tid 5 to mira0437250-235 2014-07-01 16:52:17,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 47 (ObjNum 234 snap 0 seq_num 234) dirty exists 2014-07-01 16:52:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 1 to mira0437250-236 2014-07-01 16:52:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 2 to mira0437250-236 2014-07-01 16:52:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 3 to mira0437250-236 2014-07-01 16:52:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:235: done (15 left) 2014-07-01 16:52:17,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: write initial oid 251 2014-07-01 16:52:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: seq_num 250 ranges {561870=487885,1769000=392870} 2014-07-01 16:52:17,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0437250-251 from 561870 to 1049755 tid 1 2014-07-01 16:52:17,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: writing mira0437250-251 from 1769000 to 2161870 tid 2 2014-07-01 16:52:17,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 4 to mira0437250-236 2014-07-01 16:52:17,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 6 to mira0437250-236 2014-07-01 16:52:17,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: finishing write tid 7 to mira0437250-236 2014-07-01 16:52:17,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 36 (ObjNum 235 snap 0 seq_num 235) dirty exists 2014-07-01 16:52:17,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 1 to mira0437250-237 2014-07-01 16:52:17,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 2 to mira0437250-237 2014-07-01 16:52:17,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 3 to mira0437250-237 2014-07-01 16:52:17,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:236: done (15 left) 2014-07-01 16:52:17,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: write initial oid 252 2014-07-01 16:52:17,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: seq_num 251 ranges {506700=494761,1766780=580189,2899004=7696} 2014-07-01 16:52:17,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0437250-252 from 506700 to 1001461 tid 1 2014-07-01 16:52:17,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0437250-252 from 1766780 to 2346969 tid 2 2014-07-01 16:52:17,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: writing mira0437250-252 from 2899004 to 2906700 tid 3 2014-07-01 16:52:17,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 5 to mira0437250-237 2014-07-01 16:52:17,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: finishing write tid 6 to mira0437250-237 2014-07-01 16:52:17,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 64 (ObjNum 236 snap 0 seq_num 236) dirty exists 2014-07-01 16:52:17,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 1 to mira0437250-238 2014-07-01 16:52:17,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 2 to mira0437250-238 2014-07-01 16:52:17,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 3 to mira0437250-238 2014-07-01 16:52:17,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 5 to mira0437250-238 2014-07-01 16:52:17,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: finishing write tid 6 to mira0437250-238 2014-07-01 16:52:17,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 43 (ObjNum 237 snap 0 seq_num 237) dirty exists 2014-07-01 16:52:17,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 1 to mira0437250-239 2014-07-01 16:52:17,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:237: done (15 left) 2014-07-01 16:52:17,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:238: done (14 left) 2014-07-01 16:52:17,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: write initial oid 253 2014-07-01 16:52:17,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: seq_num 252 ranges {451529=466565,1578063=400784,2699204=555493,3651528=1} 2014-07-01 16:52:17,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0437250-253 from 451529 to 918094 tid 1 2014-07-01 16:52:17,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0437250-253 from 1578063 to 1978847 tid 2 2014-07-01 16:52:17,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0437250-253 from 2699204 to 3254697 tid 3 2014-07-01 16:52:17,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: writing mira0437250-253 from 3651528 to 3651529 tid 4 2014-07-01 16:52:17,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: write initial oid 254 2014-07-01 16:52:17,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: seq_num 253 ranges {663111=589794,1675300=754627,2900430=688812,3863110=1} 2014-07-01 16:52:17,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0437250-254 from 663111 to 1252905 tid 1 2014-07-01 16:52:17,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0437250-254 from 1675300 to 2429927 tid 2 2014-07-01 16:52:17,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0437250-254 from 2900430 to 3589242 tid 3 2014-07-01 16:52:17,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: writing mira0437250-254 from 3863110 to 3863111 tid 4 2014-07-01 16:52:17,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 3 to mira0437250-239 2014-07-01 16:52:17,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: finishing write tid 4 to mira0437250-239 2014-07-01 16:52:17,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 52 (ObjNum 238 snap 0 seq_num 238) dirty exists 2014-07-01 16:52:17,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 1 to mira0437250-240 2014-07-01 16:52:17,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:239: done (15 left) 2014-07-01 16:52:17,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: write initial oid 255 2014-07-01 16:52:17,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: seq_num 254 ranges {607940=1} 2014-07-01 16:52:17,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: writing mira0437250-255 from 607940 to 607941 tid 1 2014-07-01 16:52:17,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 3 to mira0437250-240 2014-07-01 16:52:17,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 1 to mira0437250-241 2014-07-01 16:52:17,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 3 to mira0437250-241 2014-07-01 16:52:17,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: finishing write tid 4 to mira0437250-241 2014-07-01 16:52:17,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 49 (ObjNum 240 snap 0 seq_num 240) dirty exists 2014-07-01 16:52:17,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: finishing write tid 4 to mira0437250-240 2014-07-01 16:52:17,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 54 (ObjNum 239 snap 0 seq_num 239) dirty exists 2014-07-01 16:52:17,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:240: done (15 left) 2014-07-01 16:52:17,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:241: done (14 left) 2014-07-01 16:52:17,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: write initial oid 256 2014-07-01 16:52:17,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: seq_num 255 ranges {550723=418074,1350722=1} 2014-07-01 16:52:17,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: writing mira0437250-256 from 550723 to 968797 tid 1 2014-07-01 16:52:17,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: writing mira0437250-256 from 1350722 to 1350723 tid 2 2014-07-01 16:52:17,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: write initial oid 257 2014-07-01 16:52:17,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: seq_num 256 ranges {646976=400000} 2014-07-01 16:52:17,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: writing mira0437250-257 from 646976 to 1046976 tid 1 2014-07-01 16:52:17,522.522 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:52:17,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,533.533 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:52:17,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 1 to mira0437250-242 2014-07-01 16:52:17,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 1 to mira0437250-244 2014-07-01 16:52:17,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 2 to mira0437250-244 2014-07-01 16:52:17,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 4 to mira0437250-244 2014-07-01 16:52:17,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 1 to mira0437250-253 2014-07-01 16:52:17,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 2 to mira0437250-253 2014-07-01 16:52:17,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 3 to mira0437250-253 2014-07-01 16:52:17,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 4 to mira0437250-253 2014-07-01 16:52:17,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 6 to mira0437250-253 2014-07-01 16:52:17,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 1 to mira0437250-255 2014-07-01 16:52:17,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 3 to mira0437250-255 2014-07-01 16:52:17,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 2 to mira0437250-242 2014-07-01 16:52:17,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 1 to mira0437250-246 2014-07-01 16:52:17,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 3 to mira0437250-246 2014-07-01 16:52:17,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 1 to mira0437250-248 2014-07-01 16:52:17,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: finishing write tid 5 to mira0437250-244 2014-07-01 16:52:17,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 67 (ObjNum 243 snap 0 seq_num 243) dirty exists 2014-07-01 16:52:17,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 3 to mira0437250-242 2014-07-01 16:52:17,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:244: done (15 left) 2014-07-01 16:52:17,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: write initial oid 258 2014-07-01 16:52:17,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: seq_num 257 ranges {459582=514131,1259581=1} 2014-07-01 16:52:17,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0437250-258 from 459582 to 973713 tid 1 2014-07-01 16:52:17,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: writing mira0437250-258 from 1259581 to 1259582 tid 2 2014-07-01 16:52:17,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 5 to mira0437250-242 2014-07-01 16:52:17,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 3 to mira0437250-248 2014-07-01 16:52:17,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 1 to mira0437250-243 2014-07-01 16:52:17,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 2 to mira0437250-243 2014-07-01 16:52:17,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 3 to mira0437250-243 2014-07-01 16:52:17,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 5 to mira0437250-243 2014-07-01 16:52:17,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 1 to mira0437250-247 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 2 to mira0437250-247 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 3 to mira0437250-247 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: finishing write tid 4 to mira0437250-248 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 53 (ObjNum 247 snap 0 seq_num 247) dirty exists 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 1 to mira0437250-245 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: finishing write tid 4 to mira0437250-246 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 51 (ObjNum 245 snap 0 seq_num 245) dirty exists 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 4 to mira0437250-247 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 6 to mira0437250-247 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 2 to mira0437250-245 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 3 to mira0437250-245 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 5 to mira0437250-245 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 1 to mira0437250-250 2014-07-01 16:52:17,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 2 to mira0437250-250 2014-07-01 16:52:17,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 4 to mira0437250-250 2014-07-01 16:52:17,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: finishing write tid 6 to mira0437250-245 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 245 v 40 (ObjNum 244 snap 0 seq_num 244) dirty exists 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 1 to mira0437250-249 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 2 to mira0437250-249 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 4 to mira0437250-249 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 1 to mira0437250-257 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 3 to mira0437250-257 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 1 to mira0437250-251 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 2 to mira0437250-251 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:245: done (15 left) 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:246: done (14 left) 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:248: done (13 left) 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: write initial oid 259 2014-07-01 16:52:17,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: seq_num 258 ranges {651964=485935,1719496=608451,3017212=34752} 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: writing mira0437250-259 from 651964 to 1137899 tid 1 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: writing mira0437250-259 from 1719496 to 2327947 tid 2 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: writing mira0437250-259 from 3017212 to 3051964 tid 3 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: write initial oid 260 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: seq_num 259 ranges {748218=492812,1868701=678646,2748217=1} 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0437250-260 from 748218 to 1241030 tid 1 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0437250-260 from 1868701 to 2547347 tid 2 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: writing mira0437250-260 from 2748217 to 2748218 tid 3 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: write initial oid 261 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: seq_num 260 ranges {443447=464616,1698160=745287} 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: writing mira0437250-261 from 443447 to 908063 tid 1 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: writing mira0437250-261 from 1698160 to 2443447 tid 2 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 1 to mira0437250-252 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 4 to mira0437250-251 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: finishing write tid 6 to mira0437250-242 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 41 (ObjNum 241 snap 0 seq_num 241) dirty exists 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 2 to mira0437250-252 2014-07-01 16:52:17,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 3 to mira0437250-252 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 5 to mira0437250-252 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:242: done (15 left) 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: write initial oid 262 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: seq_num 261 ranges {236852=1} 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: writing mira0437250-262 from 236852 to 236853 tid 1 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 1 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 2 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 3 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 4 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 6 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: finishing write tid 7 to mira0437250-254 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 54 (ObjNum 253 snap 0 seq_num 253) dirty exists 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 1 to mira0437250-256 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 2 to mira0437250-256 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 4 to mira0437250-256 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: finishing write tid 6 to mira0437250-252 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 56 (ObjNum 251 snap 0 seq_num 251) dirty exists 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: finishing write tid 5 to mira0437250-256 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 256 v 57 (ObjNum 255 snap 0 seq_num 255) dirty exists 2014-07-01 16:52:17,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: finishing write tid 5 to mira0437250-251 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 57 (ObjNum 250 snap 0 seq_num 250) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: finishing write tid 4 to mira0437250-255 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 255 v 54 (ObjNum 254 snap 0 seq_num 254) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: finishing write tid 6 to mira0437250-243 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 40 (ObjNum 242 snap 0 seq_num 242) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: finishing write tid 5 to mira0437250-249 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 43 (ObjNum 248 snap 0 seq_num 248) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: finishing write tid 7 to mira0437250-253 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 72 (ObjNum 252 snap 0 seq_num 252) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: finishing write tid 7 to mira0437250-247 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 48 (ObjNum 246 snap 0 seq_num 246) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: finishing write tid 5 to mira0437250-250 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 51 (ObjNum 249 snap 0 seq_num 249) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 1 to mira0437250-258 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 2 to mira0437250-258 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 4 to mira0437250-258 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: finishing write tid 5 to mira0437250-258 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 258 v 60 (ObjNum 257 snap 0 seq_num 257) dirty exists 2014-07-01 16:52:17,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: finishing write tid 4 to mira0437250-257 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 257 v 42 (ObjNum 256 snap 0 seq_num 256) dirty exists 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 1 to mira0437250-259 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 2 to mira0437250-259 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 3 to mira0437250-259 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 5 to mira0437250-259 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: finishing write tid 6 to mira0437250-259 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 46 (ObjNum 258 snap 0 seq_num 258) dirty exists 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 1 to mira0437250-260 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 2 to mira0437250-260 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:243: done (15 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:247: done (14 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:249: done (13 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:250: done (12 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:251: done (11 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:252: done (10 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:253: done (9 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:254: done (8 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:255: done (7 left) 2014-07-01 16:52:18,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:256: done (6 left) 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:257: done (5 left) 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:258: done (4 left) 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:259: done (3 left) 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: write initial oid 263 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: seq_num 262 ranges {449458=1} 2014-07-01 16:52:18,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: expect (ObjNum 1534 snap 0 seq_num 1534) 2014-07-01 16:52:18,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: writing mira0437250-263 from 449458 to 449459 tid 1 2014-07-01 16:52:18,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: write initial oid 264 2014-07-01 16:52:18,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: seq_num 263 ranges {143664=1} 2014-07-01 16:52:18,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: writing mira0437250-264 from 143664 to 143665 tid 1 2014-07-01 16:52:18,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: write initial oid 265 2014-07-01 16:52:18,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: seq_num 264 ranges {737070=788953,1937069=1} 2014-07-01 16:52:18,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: writing mira0437250-265 from 737070 to 1526023 tid 1 2014-07-01 16:52:18,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: writing mira0437250-265 from 1937069 to 1937070 tid 2 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: write initial oid 266 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: seq_num 265 ranges {432300=512182,1493782=138518} 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0437250-266 from 432300 to 944482 tid 1 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: writing mira0437250-266 from 1493782 to 1632300 tid 2 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: write initial oid 267 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: seq_num 266 ranges {644906=483986,1724266=120640} 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: writing mira0437250-267 from 644906 to 1128892 tid 1 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: writing mira0437250-267 from 1724266 to 1844906 tid 2 2014-07-01 16:52:18,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: write initial oid 268 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: seq_num 267 ranges {589735=490862,1722045=626704,2589734=1} 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0437250-268 from 589735 to 1080597 tid 1 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0437250-268 from 1722045 to 2348749 tid 2 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: writing mira0437250-268 from 2589734 to 2589735 tid 3 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: write initial oid 269 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: seq_num 268 ranges {533541=463691,1649682=448323,2742491=591050} 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0437250-269 from 533541 to 997232 tid 1 2014-07-01 16:52:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0437250-269 from 1649682 to 2098005 tid 2 2014-07-01 16:52:18,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: writing mira0437250-269 from 2742491 to 3333541 tid 3 2014-07-01 16:52:18,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: write initial oid 270 2014-07-01 16:52:18,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: seq_num 269 ranges {629795=586919,1915238=669942,3029794=1} 2014-07-01 16:52:18,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: writing mira0437250-270 from 629795 to 1216714 tid 1 2014-07-01 16:52:18,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: writing mira0437250-270 from 1915238 to 2585180 tid 2 2014-07-01 16:52:18,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: writing mira0437250-270 from 3029794 to 3029795 tid 3 2014-07-01 16:52:18,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: write initial oid 271 2014-07-01 16:52:18,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: seq_num 270 ranges {573601=558723,1593274=588713,2709869=719323,3773600=1} 2014-07-01 16:52:18,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: writing mira0437250-271 from 573601 to 1132324 tid 1 2014-07-01 16:52:18,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: writing mira0437250-271 from 1593274 to 2181987 tid 2 2014-07-01 16:52:18,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: writing mira0437250-271 from 2709869 to 3429192 tid 3 2014-07-01 16:52:18,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: writing mira0437250-271 from 3773600 to 3773601 tid 4 2014-07-01 16:52:18,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: write initial oid 272 2014-07-01 16:52:18,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: seq_num 271 ranges {633758=400000} 2014-07-01 16:52:18,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: writing mira0437250-272 from 633758 to 1033758 tid 1 2014-07-01 16:52:18,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: write initial oid 273 2014-07-01 16:52:18,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: seq_num 272 ranges {730011=1} 2014-07-01 16:52:18,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: writing mira0437250-273 from 730011 to 730012 tid 1 2014-07-01 16:52:18,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: write initial oid 274 2014-07-01 16:52:18,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: seq_num 273 ranges {673818=661656,1473817=1} 2014-07-01 16:52:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0437250-274 from 673818 to 1335474 tid 1 2014-07-01 16:52:18,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: writing mira0437250-274 from 1473817 to 1473818 tid 2 2014-07-01 16:52:18,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: write initial oid 275 2014-07-01 16:52:18,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: seq_num 274 ranges {618648=634485,1710859=507789} 2014-07-01 16:52:18,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: writing mira0437250-275 from 618648 to 1253133 tid 1 2014-07-01 16:52:18,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: writing mira0437250-275 from 1710859 to 2218648 tid 2 2014-07-01 16:52:18,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 3 to mira0437250-260 2014-07-01 16:52:18,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 5 to mira0437250-260 2014-07-01 16:52:18,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: finishing write tid 6 to mira0437250-260 2014-07-01 16:52:18,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 61 (ObjNum 259 snap 0 seq_num 259) dirty exists 2014-07-01 16:52:18,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 1 to mira0437250-261 2014-07-01 16:52:18,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:260: done (15 left) 2014-07-01 16:52:18,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: write initial oid 276 2014-07-01 16:52:18,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: seq_num 275 ranges {430229=490961,1541342=725161,2430228=1} 2014-07-01 16:52:18,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0437250-276 from 430229 to 921190 tid 1 2014-07-01 16:52:18,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0437250-276 from 1541342 to 2266503 tid 2 2014-07-01 16:52:18,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: writing mira0437250-276 from 2430228 to 2430229 tid 3 2014-07-01 16:52:18,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 1 to mira0437250-262 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 1 to mira0437250-263 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 3 to mira0437250-263 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: finishing write tid 4 to mira0437250-263 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 49 (ObjNum 262 snap 0 seq_num 262) dirty exists 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 3 to mira0437250-262 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: finishing write tid 4 to mira0437250-262 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 62 (ObjNum 261 snap 0 seq_num 261) dirty exists 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 2 to mira0437250-261 2014-07-01 16:52:18,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 4 to mira0437250-261 2014-07-01 16:52:18,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: finishing write tid 5 to mira0437250-261 2014-07-01 16:52:18,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 49 (ObjNum 260 snap 0 seq_num 260) dirty exists 2014-07-01 16:52:18,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 1 to mira0437250-264 2014-07-01 16:52:18,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 3 to mira0437250-264 2014-07-01 16:52:18,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: finishing write tid 4 to mira0437250-264 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 64 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 1 to mira0437250-265 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 2 to mira0437250-265 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:261: done (15 left) 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:262: done (14 left) 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:263: done (13 left) 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:264: done (12 left) 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: write initial oid 277 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: seq_num 276 ranges {775059=462765,1904050=546780,3098021=77038} 2014-07-01 16:52:18,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0437250-277 from 775059 to 1237824 tid 1 2014-07-01 16:52:18,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0437250-277 from 1904050 to 2450830 tid 2 2014-07-01 16:52:18,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: writing mira0437250-277 from 3098021 to 3175059 tid 3 2014-07-01 16:52:18,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: write initial oid 278 2014-07-01 16:52:18,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: seq_num 277 ranges {719889=585994,1866759=616975,3031471=648264,3919888=1} 2014-07-01 16:52:18,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0437250-278 from 719889 to 1305883 tid 1 2014-07-01 16:52:18,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0437250-278 from 1866759 to 2483734 tid 2 2014-07-01 16:52:18,535.535 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:52:18,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0437250-278 from 3031471 to 3679735 tid 3 2014-07-01 16:52:18,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: writing mira0437250-278 from 3919888 to 3919889 tid 4 2014-07-01 16:52:18,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: write initial oid 279 2014-07-01 16:52:18,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: seq_num 278 ranges {415119=558822,1697244=687171,2798650=535055,3615118=1} 2014-07-01 16:52:18,550.550 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:52:18,550.550 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:52:18,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: writing mira0437250-279 from 415119 to 973941 tid 1 2014-07-01 16:52:18,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: writing mira0437250-279 from 1697244 to 2384415 tid 2 2014-07-01 16:52:18,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: writing mira0437250-279 from 2798650 to 3333705 tid 3 2014-07-01 16:52:18,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: writing mira0437250-279 from 3615118 to 3615119 tid 4 2014-07-01 16:52:18,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: write initial oid 280 2014-07-01 16:52:18,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: seq_num 279 ranges {358923=1} 2014-07-01 16:52:18,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: writing mira0437250-280 from 358923 to 358924 tid 1 2014-07-01 16:52:18,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 4 to mira0437250-265 2014-07-01 16:52:18,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: finishing write tid 5 to mira0437250-265 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 265 v 47 (ObjNum 264 snap 0 seq_num 264) dirty exists 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 1 to mira0437250-266 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 2 to mira0437250-266 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 4 to mira0437250-266 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: finishing write tid 5 to mira0437250-266 2014-07-01 16:52:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 52 (ObjNum 265 snap 0 seq_num 265) dirty exists 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 1 to mira0437250-267 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 2 to mira0437250-267 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 4 to mira0437250-267 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: finishing write tid 5 to mira0437250-267 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 54 (ObjNum 266 snap 0 seq_num 266) dirty exists 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 1 to mira0437250-268 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 2 to mira0437250-268 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 3 to mira0437250-268 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 5 to mira0437250-268 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: finishing write tid 6 to mira0437250-268 2014-07-01 16:52:18,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 57 (ObjNum 267 snap 0 seq_num 267) dirty exists 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 1 to mira0437250-269 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 2 to mira0437250-269 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 1 to mira0437250-270 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 2 to mira0437250-270 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 3 to mira0437250-270 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 1 to mira0437250-271 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:265: done (15 left) 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:266: done (14 left) 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:267: done (13 left) 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:268: done (12 left) 2014-07-01 16:52:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: write initial oid 281 2014-07-01 16:52:18,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: seq_num 280 ranges {570505=1} 2014-07-01 16:52:18,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: writing mira0437250-281 from 570505 to 570506 tid 1 2014-07-01 16:52:18,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: write initial oid 282 2014-07-01 16:52:18,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: seq_num 281 ranges {515336=660731,1315335=1} 2014-07-01 16:52:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: writing mira0437250-282 from 515336 to 1176067 tid 1 2014-07-01 16:52:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: writing mira0437250-282 from 1315335 to 1315336 tid 2 2014-07-01 16:52:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: write initial oid 283 2014-07-01 16:52:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: seq_num 282 ranges {460166=633560,1813805=246361} 2014-07-01 16:52:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: writing mira0437250-283 from 460166 to 1093726 tid 1 2014-07-01 16:52:18,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: writing mira0437250-283 from 1813805 to 2060166 tid 2 2014-07-01 16:52:18,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: write initial oid 284 2014-07-01 16:52:18,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: seq_num 283 ranges {555395=640436,1755394=1} 2014-07-01 16:52:18,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: writing mira0437250-284 from 555395 to 1195831 tid 1 2014-07-01 16:52:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: writing mira0437250-284 from 1755394 to 1755395 tid 2 2014-07-01 16:52:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 2 to mira0437250-271 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 3 to mira0437250-271 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 4 to mira0437250-271 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 6 to mira0437250-271 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 5 to mira0437250-270 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 1 to mira0437250-273 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 3 to mira0437250-273 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 1 to mira0437250-275 2014-07-01 16:52:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 2 to mira0437250-275 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 1 to mira0437250-277 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 4 to mira0437250-275 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 2 to mira0437250-277 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: finishing write tid 6 to mira0437250-270 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 270 v 76 (ObjNum 269 snap 0 seq_num 269) dirty exists 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:270: done (15 left) 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: write initial oid 285 2014-07-01 16:52:18,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: seq_num 284 ranges {500225=612240,1889620=494838,2500224=1} 2014-07-01 16:52:18,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: writing mira0437250-285 from 500225 to 1112465 tid 1 2014-07-01 16:52:18,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: writing mira0437250-285 from 1889620 to 2384458 tid 2 2014-07-01 16:52:18,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: writing mira0437250-285 from 2500224 to 2500225 tid 3 2014-07-01 16:52:18,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: finishing write tid 7 to mira0437250-271 2014-07-01 16:52:18,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 271 v 65 (ObjNum 270 snap 0 seq_num 270) dirty exists 2014-07-01 16:52:18,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 3 to mira0437250-269 2014-07-01 16:52:18,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:271: done (15 left) 2014-07-01 16:52:18,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: write initial oid 286 2014-07-01 16:52:18,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: seq_num 285 ranges {560383=736493,1836457=716457,3103355=578301,3760382=1} 2014-07-01 16:52:18,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0437250-286 from 560383 to 1296876 tid 1 2014-07-01 16:52:18,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0437250-286 from 1836457 to 2552914 tid 2 2014-07-01 16:52:18,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0437250-286 from 3103355 to 3681656 tid 3 2014-07-01 16:52:18,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: writing mira0437250-286 from 3760382 to 3760383 tid 4 2014-07-01 16:52:18,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: finishing write tid 4 to mira0437250-273 2014-07-01 16:52:18,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 78 (ObjNum 272 snap 0 seq_num 272) dirty exists 2014-07-01 16:52:18,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 3 to mira0437250-277 2014-07-01 16:52:18,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 5 to mira0437250-277 2014-07-01 16:52:18,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 1 to mira0437250-278 2014-07-01 16:52:18,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 2 to mira0437250-278 2014-07-01 16:52:18,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 3 to mira0437250-278 2014-07-01 16:52:18,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 4 to mira0437250-278 2014-07-01 16:52:18,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 6 to mira0437250-278 2014-07-01 16:52:18,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 1 to mira0437250-284 2014-07-01 16:52:18,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:273: done (15 left) 2014-07-01 16:52:18,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: write initial oid 287 2014-07-01 16:52:18,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: seq_num 286 ranges {656636=556873,1647740=786652,3134979=321657} 2014-07-01 16:52:18,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: writing mira0437250-287 from 656636 to 1213509 tid 1 2014-07-01 16:52:18,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: writing mira0437250-287 from 1647740 to 2434392 tid 2 2014-07-01 16:52:18,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: writing mira0437250-287 from 3134979 to 3456636 tid 3 2014-07-01 16:52:18,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:18,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 2 to mira0437250-284 2014-07-01 16:52:18,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 5 to mira0437250-269 2014-07-01 16:52:18,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 1 to mira0437250-272 2014-07-01 16:52:18,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 3 to mira0437250-272 2014-07-01 16:52:18,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 1 to mira0437250-274 2014-07-01 16:52:18,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 1 to mira0437250-280 2014-07-01 16:52:18,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 3 to mira0437250-280 2014-07-01 16:52:18,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 1 to mira0437250-282 2014-07-01 16:52:18,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 2 to mira0437250-282 2014-07-01 16:52:18,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 4 to mira0437250-282 2014-07-01 16:52:18,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 1 to mira0437250-283 2014-07-01 16:52:18,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 2 to mira0437250-283 2014-07-01 16:52:18,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 1 to mira0437250-276 2014-07-01 16:52:18,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 2 to mira0437250-274 2014-07-01 16:52:18,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 2 to mira0437250-276 2014-07-01 16:52:18,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: finishing write tid 5 to mira0437250-275 2014-07-01 16:52:18,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 275 v 57 (ObjNum 274 snap 0 seq_num 274) dirty exists 2014-07-01 16:52:18,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 4 to mira0437250-283 2014-07-01 16:52:18,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 4 to mira0437250-284 2014-07-01 16:52:18,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 3 to mira0437250-276 2014-07-01 16:52:18,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 5 to mira0437250-276 2014-07-01 16:52:18,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: finishing write tid 6 to mira0437250-276 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 276 v 47 (ObjNum 275 snap 0 seq_num 275) dirty exists 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 4 to mira0437250-274 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: finishing write tid 6 to mira0437250-277 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 51 (ObjNum 276 snap 0 seq_num 276) dirty exists 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: finishing write tid 5 to mira0437250-274 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 65 (ObjNum 273 snap 0 seq_num 273) dirty exists 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 1 to mira0437250-279 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: finishing write tid 6 to mira0437250-269 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 269 v 45 (ObjNum 268 snap 0 seq_num 268) dirty exists 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 2 to mira0437250-279 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: finishing write tid 4 to mira0437250-272 2014-07-01 16:52:18,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 56 (ObjNum 271 snap 0 seq_num 271) dirty exists 2014-07-01 16:52:18,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 1 to mira0437250-285 2014-07-01 16:52:18,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 3 to mira0437250-279 2014-07-01 16:52:18,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 4 to mira0437250-279 2014-07-01 16:52:18,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 6 to mira0437250-279 2014-07-01 16:52:18,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 1 to mira0437250-281 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 3 to mira0437250-281 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: finishing write tid 7 to mira0437250-279 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 70 (ObjNum 278 snap 0 seq_num 278) dirty exists 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: finishing write tid 4 to mira0437250-281 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 63 (ObjNum 280 snap 0 seq_num 280) dirty exists 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: finishing write tid 7 to mira0437250-278 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 56 (ObjNum 277 snap 0 seq_num 277) dirty exists 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: finishing write tid 4 to mira0437250-280 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 47 (ObjNum 279 snap 0 seq_num 279) dirty exists 2014-07-01 16:52:18,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 2 to mira0437250-285 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: finishing write tid 5 to mira0437250-284 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 60 (ObjNum 283 snap 0 seq_num 283) dirty exists 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 3 to mira0437250-285 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 5 to mira0437250-285 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: finishing write tid 5 to mira0437250-282 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 67 (ObjNum 281 snap 0 seq_num 281) dirty exists 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: finishing write tid 5 to mira0437250-283 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 70 (ObjNum 282 snap 0 seq_num 282) dirty exists 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: finishing write tid 6 to mira0437250-285 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 51 (ObjNum 284 snap 0 seq_num 284) dirty exists 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 1 to mira0437250-286 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 2 to mira0437250-286 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 3 to mira0437250-286 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 4 to mira0437250-286 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:269: done (15 left) 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:272: done (14 left) 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:274: done (13 left) 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:275: done (12 left) 2014-07-01 16:52:18,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:276: done (11 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:277: done (10 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:278: done (9 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:279: done (8 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:280: done (7 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:281: done (6 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:282: done (5 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:283: done (4 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:284: done (3 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:285: done (2 left) 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: write initial oid 288 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: seq_num 287 ranges {201465=1} 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: writing mira0437250-288 from 201465 to 201466 tid 1 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: write initial oid 289 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: seq_num 288 ranges {544248=400000} 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: writing mira0437250-289 from 544248 to 944248 tid 1 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: write initial oid 290 2014-07-01 16:52:18,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: seq_num 289 ranges {756854=400000} 2014-07-01 16:52:18,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: writing mira0437250-290 from 756854 to 1156854 tid 1 2014-07-01 16:52:18,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: write initial oid 291 2014-07-01 16:52:18,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: seq_num 290 ranges {701683=631610,1901682=1} 2014-07-01 16:52:18,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0437250-291 from 701683 to 1333293 tid 1 2014-07-01 16:52:18,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: writing mira0437250-291 from 1901682 to 1901683 tid 2 2014-07-01 16:52:18,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: write initial oid 292 2014-07-01 16:52:18,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: seq_num 291 ranges {645489=638487,2028833=524124,2645488=1} 2014-07-01 16:52:18,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0437250-292 from 645489 to 1283976 tid 1 2014-07-01 16:52:18,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0437250-292 from 2028833 to 2552957 tid 2 2014-07-01 16:52:18,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: writing mira0437250-292 from 2645488 to 2645489 tid 3 2014-07-01 16:52:18,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: write initial oid 293 2014-07-01 16:52:18,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: seq_num 292 ranges {741743=610291,1991541=350202} 2014-07-01 16:52:19,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: writing mira0437250-293 from 741743 to 1352034 tid 1 2014-07-01 16:52:19,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: writing mira0437250-293 from 1991541 to 2341743 tid 2 2014-07-01 16:52:19,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: write initial oid 294 2014-07-01 16:52:19,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: seq_num 293 ranges {685549=734543,1822025=414914,2673732=411817} 2014-07-01 16:52:19,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: writing mira0437250-294 from 685549 to 1420092 tid 1 2014-07-01 16:52:19,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: writing mira0437250-294 from 1822025 to 2236939 tid 2 2014-07-01 16:52:19,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: writing mira0437250-294 from 2673732 to 3085549 tid 3 2014-07-01 16:52:19,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: write initial oid 295 2014-07-01 16:52:19,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: seq_num 294 ranges {346729=1} 2014-07-01 16:52:19,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: writing mira0437250-295 from 346729 to 346730 tid 1 2014-07-01 16:52:19,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: write initial oid 296 2014-07-01 16:52:19,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: seq_num 295 ranges {42983=1} 2014-07-01 16:52:19,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: writing mira0437250-296 from 42983 to 42984 tid 1 2014-07-01 16:52:19,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: write initial oid 297 2014-07-01 16:52:19,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: seq_num 296 ranges {785765=1} 2014-07-01 16:52:19,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: writing mira0437250-297 from 785765 to 785766 tid 1 2014-07-01 16:52:19,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: write initial oid 298 2014-07-01 16:52:19,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: seq_num 297 ranges {730596=657857,1530595=1} 2014-07-01 16:52:19,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0437250-298 from 730596 to 1388453 tid 1 2014-07-01 16:52:19,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: writing mira0437250-298 from 1530595 to 1530596 tid 2 2014-07-01 16:52:19,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: write initial oid 299 2014-07-01 16:52:19,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: seq_num 298 ranges {542177=629661,1732974=9203} 2014-07-01 16:52:19,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: writing mira0437250-299 from 542177 to 1171838 tid 1 2014-07-01 16:52:19,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: writing mira0437250-299 from 1732974 to 1742177 tid 2 2014-07-01 16:52:19,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: write initial oid 300 2014-07-01 16:52:19,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: seq_num 299 ranges {487007=636537,1730754=622582,2487006=1} 2014-07-01 16:52:19,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0437250-300 from 487007 to 1123544 tid 1 2014-07-01 16:52:19,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0437250-300 from 1730754 to 2353336 tid 2 2014-07-01 16:52:19,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: writing mira0437250-300 from 2487006 to 2487007 tid 3 2014-07-01 16:52:19,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: write initial oid 301 2014-07-01 16:52:19,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 6 to mira0437250-286 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: finishing write tid 7 to mira0437250-286 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 75 (ObjNum 285 snap 0 seq_num 285) dirty exists 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 1 to mira0437250-287 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 2 to mira0437250-287 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: seq_num 300 ranges {431837=609366,1810839=692777,3077640=154197} 2014-07-01 16:52:19,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0437250-301 from 431837 to 1041203 tid 1 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0437250-301 from 1810839 to 2503616 tid 2 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: writing mira0437250-301 from 3077640 to 3231837 tid 3 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:286: done (15 left) 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: write initial oid 302 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: seq_num 301 ranges {527067=732594,1675371=762972,2927066=1} 2014-07-01 16:52:19,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: writing mira0437250-302 from 527067 to 1259661 tid 1 2014-07-01 16:52:19,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: writing mira0437250-302 from 1675371 to 2438343 tid 2 2014-07-01 16:52:19,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: writing mira0437250-302 from 2927066 to 2927067 tid 3 2014-07-01 16:52:19,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 1 to mira0437250-288 2014-07-01 16:52:19,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 3 to mira0437250-288 2014-07-01 16:52:19,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 3 to mira0437250-287 2014-07-01 16:52:19,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 5 to mira0437250-287 2014-07-01 16:52:19,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: finishing write tid 4 to mira0437250-288 2014-07-01 16:52:19,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 58 (ObjNum 287 snap 0 seq_num 287) dirty exists 2014-07-01 16:52:19,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: finishing write tid 6 to mira0437250-287 2014-07-01 16:52:19,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 60 (ObjNum 286 snap 0 seq_num 286) dirty exists 2014-07-01 16:52:19,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 1 to mira0437250-289 2014-07-01 16:52:19,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 1 to mira0437250-291 2014-07-01 16:52:19,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 1 to mira0437250-293 2014-07-01 16:52:19,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:287: done (15 left) 2014-07-01 16:52:19,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:288: done (14 left) 2014-07-01 16:52:19,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: write initial oid 303 2014-07-01 16:52:19,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: seq_num 302 ranges {471896=704398,1886654=584591,3211289=458411,3671895=1} 2014-07-01 16:52:19,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: writing mira0437250-303 from 471896 to 1176294 tid 1 2014-07-01 16:52:19,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: writing mira0437250-303 from 1886654 to 2471245 tid 2 2014-07-01 16:52:19,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: writing mira0437250-303 from 3211289 to 3669700 tid 3 2014-07-01 16:52:19,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: writing mira0437250-303 from 3671895 to 3671896 tid 4 2014-07-01 16:52:19,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: write initial oid 304 2014-07-01 16:52:19,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: seq_num 303 ranges {532054=400000} 2014-07-01 16:52:19,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: writing mira0437250-304 from 532054 to 932054 tid 1 2014-07-01 16:52:19,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 3 to mira0437250-289 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 1 to mira0437250-290 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 3 to mira0437250-290 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 1 to mira0437250-292 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 2 to mira0437250-292 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 3 to mira0437250-292 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 5 to mira0437250-292 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 1 to mira0437250-294 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 2 to mira0437250-294 2014-07-01 16:52:19,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 3 to mira0437250-294 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 1 to mira0437250-298 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 2 to mira0437250-298 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 4 to mira0437250-298 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 2 to mira0437250-293 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 2 to mira0437250-291 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 5 to mira0437250-294 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 4 to mira0437250-293 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 4 to mira0437250-291 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 1 to mira0437250-295 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 3 to mira0437250-295 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 1 to mira0437250-297 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 3 to mira0437250-297 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: finishing write tid 6 to mira0437250-294 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 294 v 60 (ObjNum 293 snap 0 seq_num 293) dirty exists 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: finishing write tid 5 to mira0437250-291 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 60 (ObjNum 290 snap 0 seq_num 290) dirty exists 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 1 to mira0437250-296 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 3 to mira0437250-296 2014-07-01 16:52:19,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: finishing write tid 4 to mira0437250-289 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 49 (ObjNum 288 snap 0 seq_num 288) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: finishing write tid 4 to mira0437250-296 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 67 (ObjNum 295 snap 0 seq_num 295) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 1 to mira0437250-299 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 2 to mira0437250-299 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 4 to mira0437250-299 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: finishing write tid 4 to mira0437250-290 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 53 (ObjNum 289 snap 0 seq_num 289) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: finishing write tid 5 to mira0437250-293 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 52 (ObjNum 292 snap 0 seq_num 292) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: finishing write tid 4 to mira0437250-295 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 54 (ObjNum 294 snap 0 seq_num 294) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: finishing write tid 5 to mira0437250-298 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 50 (ObjNum 297 snap 0 seq_num 297) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: finishing write tid 4 to mira0437250-297 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 297 v 60 (ObjNum 296 snap 0 seq_num 296) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: finishing write tid 6 to mira0437250-292 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 79 (ObjNum 291 snap 0 seq_num 291) dirty exists 2014-07-01 16:52:19,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 1 to mira0437250-300 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: finishing write tid 5 to mira0437250-299 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 70 (ObjNum 298 snap 0 seq_num 298) dirty exists 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 2 to mira0437250-300 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 3 to mira0437250-300 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 5 to mira0437250-300 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: finishing write tid 6 to mira0437250-300 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 83 (ObjNum 299 snap 0 seq_num 299) dirty exists 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 1 to mira0437250-301 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 2 to mira0437250-301 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:289: done (15 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:290: done (14 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:291: done (13 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:292: done (12 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:293: done (11 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:294: done (10 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:295: done (9 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:296: done (8 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:297: done (7 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:298: done (6 left) 2014-07-01 16:52:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:299: done (5 left) 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:300: done (4 left) 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: write initial oid 305 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: seq_num 304 ranges {627283=1} 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: writing mira0437250-305 from 627283 to 627284 tid 1 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: write initial oid 306 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: seq_num 305 ranges {572113=408355,1372112=1} 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0437250-306 from 572113 to 980468 tid 1 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: writing mira0437250-306 from 1372112 to 1372113 tid 2 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: write initial oid 307 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: seq_num 306 ranges {667343=400000} 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: writing mira0437250-307 from 667343 to 1067343 tid 1 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: write initial oid 308 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: seq_num 307 ranges {728525=504412,1853923=474602} 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: writing mira0437250-308 from 728525 to 1232937 tid 1 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: writing mira0437250-308 from 1853923 to 2328525 tid 2 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: write initial oid 309 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: seq_num 308 ranges {672331=759864,2064183=640834,3072330=1} 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0437250-309 from 672331 to 1432195 tid 1 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0437250-309 from 2064183 to 2705017 tid 2 2014-07-01 16:52:19,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: writing mira0437250-309 from 3072330 to 3072331 tid 3 2014-07-01 16:52:19,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: write initial oid 310 2014-07-01 16:52:19,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: seq_num 309 ranges {768584=731669,2026891=462453,2768583=1} 2014-07-01 16:52:19,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: writing mira0437250-310 from 768584 to 1500253 tid 1 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: writing mira0437250-310 from 2026891 to 2489344 tid 2 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: writing mira0437250-310 from 2768583 to 2768584 tid 3 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: write initial oid 311 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: seq_num 310 ranges {713414=455921,1858399=684073,3036644=424544,3513413=1} 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: writing mira0437250-311 from 713414 to 1169335 tid 1 2014-07-01 16:52:19,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: writing mira0437250-311 from 1858399 to 2542472 tid 2 2014-07-01 16:52:19,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: writing mira0437250-311 from 3036644 to 3461188 tid 3 2014-07-01 16:52:19,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: writing mira0437250-311 from 3513413 to 3513414 tid 4 2014-07-01 16:52:19,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: write initial oid 312 2014-07-01 16:52:19,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: seq_num 311 ranges {257219=1} 2014-07-01 16:52:19,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: writing mira0437250-312 from 257219 to 257220 tid 1 2014-07-01 16:52:19,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: write initial oid 313 2014-07-01 16:52:19,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: seq_num 312 ranges {468801=1} 2014-07-01 16:52:19,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: writing mira0437250-313 from 468801 to 468802 tid 1 2014-07-01 16:52:19,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: write initial oid 314 2014-07-01 16:52:19,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: seq_num 313 ranges {412607=406406,1212606=1} 2014-07-01 16:52:19,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: writing mira0437250-314 from 412607 to 819013 tid 1 2014-07-01 16:52:19,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: writing mira0437250-314 from 1212606 to 1212607 tid 2 2014-07-01 16:52:19,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: write initial oid 315 2014-07-01 16:52:19,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: seq_num 314 ranges {757437=530659,1957436=1} 2014-07-01 16:52:19,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: writing mira0437250-315 from 757437 to 1288096 tid 1 2014-07-01 16:52:19,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: writing mira0437250-315 from 1957436 to 1957437 tid 2 2014-07-01 16:52:19,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: write initial oid 316 2014-07-01 16:52:19,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: seq_num 315 ranges {453691=502463,1404421=249270} 2014-07-01 16:52:19,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: writing mira0437250-316 from 453691 to 956154 tid 1 2014-07-01 16:52:19,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: writing mira0437250-316 from 1404421 to 1653691 tid 2 2014-07-01 16:52:19,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 3 to mira0437250-301 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 5 to mira0437250-301 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: finishing write tid 6 to mira0437250-301 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 64 (ObjNum 300 snap 0 seq_num 300) dirty exists 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 1 to mira0437250-302 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:301: done (15 left) 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: write initial oid 317 2014-07-01 16:52:19,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: seq_num 316 ranges {665273=509339,1668953=196320} 2014-07-01 16:52:19,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: writing mira0437250-317 from 665273 to 1174612 tid 1 2014-07-01 16:52:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: writing mira0437250-317 from 1668953 to 1865273 tid 2 2014-07-01 16:52:19,750.750 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:52:19,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 2 to mira0437250-302 2014-07-01 16:52:19,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 3 to mira0437250-302 2014-07-01 16:52:19,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 5 to mira0437250-302 2014-07-01 16:52:19,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: finishing write tid 6 to mira0437250-302 2014-07-01 16:52:19,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 68 (ObjNum 301 snap 0 seq_num 301) dirty exists 2014-07-01 16:52:19,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 1 to mira0437250-303 2014-07-01 16:52:19,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 2 to mira0437250-303 2014-07-01 16:52:19,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:302: done (15 left) 2014-07-01 16:52:19,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: write initial oid 318 2014-07-01 16:52:19,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: seq_num 317 ranges {458678=482168,1730861=560911,2806823=620237,3658677=1} 2014-07-01 16:52:19,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0437250-318 from 458678 to 940846 tid 1 2014-07-01 16:52:19,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0437250-318 from 1730861 to 2291772 tid 2 2014-07-01 16:52:19,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0437250-318 from 2806823 to 3427060 tid 3 2014-07-01 16:52:19,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: writing mira0437250-318 from 3658677 to 3658678 tid 4 2014-07-01 16:52:19,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 3 to mira0437250-303 2014-07-01 16:52:19,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 4 to mira0437250-303 2014-07-01 16:52:19,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 6 to mira0437250-303 2014-07-01 16:52:19,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: finishing write tid 7 to mira0437250-303 2014-07-01 16:52:19,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 88 (ObjNum 302 snap 0 seq_num 302) dirty exists 2014-07-01 16:52:19,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 1 to mira0437250-305 2014-07-01 16:52:19,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 3 to mira0437250-305 2014-07-01 16:52:19,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 1 to mira0437250-304 2014-07-01 16:52:19,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 3 to mira0437250-304 2014-07-01 16:52:19,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: finishing write tid 4 to mira0437250-305 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 80 (ObjNum 304 snap 0 seq_num 304) dirty exists 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: finishing write tid 4 to mira0437250-304 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 304 v 55 (ObjNum 303 snap 0 seq_num 303) dirty exists 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:303: done (15 left) 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:304: done (14 left) 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:305: done (13 left) 2014-07-01 16:52:19,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: write initial oid 319 2014-07-01 16:52:19,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: seq_num 318 ranges {553908=605396,1710721=782530,3273776=80132} 2014-07-01 16:52:19,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: writing mira0437250-319 from 553908 to 1159304 tid 1 2014-07-01 16:52:19,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: writing mira0437250-319 from 1710721 to 2493251 tid 2 2014-07-01 16:52:19,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: writing mira0437250-319 from 3273776 to 3353908 tid 3 2014-07-01 16:52:19,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: write initial oid 320 2014-07-01 16:52:19,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: seq_num 319 ranges {650162=425776,1673429=701301,3050161=1} 2014-07-01 16:52:19,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0437250-320 from 650162 to 1075938 tid 1 2014-07-01 16:52:19,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0437250-320 from 1673429 to 2374730 tid 2 2014-07-01 16:52:19,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: writing mira0437250-320 from 3050161 to 3050162 tid 3 2014-07-01 16:52:19,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: write initial oid 321 2014-07-01 16:52:19,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: seq_num 320 ranges {442544=400000} 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: writing mira0437250-321 from 442544 to 842544 tid 1 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 1 to mira0437250-306 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 2 to mira0437250-306 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 4 to mira0437250-306 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: finishing write tid 5 to mira0437250-306 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 63 (ObjNum 305 snap 0 seq_num 305) dirty exists 2014-07-01 16:52:19,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 1 to mira0437250-307 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 3 to mira0437250-307 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:306: done (15 left) 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: write initial oid 322 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: seq_num 321 ranges {654125=400000} 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: writing mira0437250-322 from 654125 to 1054125 tid 1 2014-07-01 16:52:19,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: finishing write tid 4 to mira0437250-307 2014-07-01 16:52:19,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 65 (ObjNum 306 snap 0 seq_num 306) dirty exists 2014-07-01 16:52:19,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 1 to mira0437250-308 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:307: done (15 left) 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: write initial oid 323 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: seq_num 322 ranges {750378=1} 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: writing mira0437250-323 from 750378 to 750379 tid 1 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 2 to mira0437250-308 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 4 to mira0437250-308 2014-07-01 16:52:19,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: finishing write tid 5 to mira0437250-308 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 57 (ObjNum 307 snap 0 seq_num 307) dirty exists 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 1 to mira0437250-309 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:308: done (15 left) 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: write initial oid 324 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: seq_num 323 ranges {542761=500514,1754918=484930,2542760=1} 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0437250-324 from 542761 to 1043275 tid 1 2014-07-01 16:52:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0437250-324 from 1754918 to 2239848 tid 2 2014-07-01 16:52:19,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: writing mira0437250-324 from 2542760 to 2542761 tid 3 2014-07-01 16:52:19,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:19,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 2 to mira0437250-309 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 3 to mira0437250-309 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 5 to mira0437250-309 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: finishing write tid 6 to mira0437250-309 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 92 (ObjNum 308 snap 0 seq_num 308) dirty exists 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 1 to mira0437250-310 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:309: done (15 left) 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: write initial oid 325 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: seq_num 324 ranges {639014=508414,1905145=333869} 2014-07-01 16:52:19,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: writing mira0437250-325 from 639014 to 1147428 tid 1 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: writing mira0437250-325 from 1905145 to 2239014 tid 2 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 2 to mira0437250-310 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 3 to mira0437250-310 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 5 to mira0437250-310 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: finishing write tid 6 to mira0437250-310 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 61 (ObjNum 309 snap 0 seq_num 309) dirty exists 2014-07-01 16:52:20,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 1 to mira0437250-311 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 2 to mira0437250-311 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 1 to mira0437250-312 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 3 to mira0437250-312 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 3 to mira0437250-311 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: finishing write tid 4 to mira0437250-312 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 70 (ObjNum 311 snap 0 seq_num 311) dirty exists 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 4 to mira0437250-311 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:310: done (15 left) 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:312: done (14 left) 2014-07-01 16:52:20,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: write initial oid 326 2014-07-01 16:52:20,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: seq_num 325 ranges {451620=480218,1451981=660392,2451619=1} 2014-07-01 16:52:20,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: writing mira0437250-326 from 451620 to 931838 tid 1 2014-07-01 16:52:20,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: writing mira0437250-326 from 1451981 to 2112373 tid 2 2014-07-01 16:52:20,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: writing mira0437250-326 from 2451619 to 2451620 tid 3 2014-07-01 16:52:20,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: write initial oid 327 2014-07-01 16:52:20,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: seq_num 326 ranges {244001=1} 2014-07-01 16:52:20,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: writing mira0437250-327 from 244001 to 244002 tid 1 2014-07-01 16:52:20,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 1 to mira0437250-313 2014-07-01 16:52:20,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 6 to mira0437250-311 2014-07-01 16:52:20,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 3 to mira0437250-313 2014-07-01 16:52:20,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: finishing write tid 7 to mira0437250-311 2014-07-01 16:52:20,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 70 (ObjNum 310 snap 0 seq_num 310) dirty exists 2014-07-01 16:52:20,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: finishing write tid 4 to mira0437250-313 2014-07-01 16:52:20,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 94 (ObjNum 312 snap 0 seq_num 312) dirty exists 2014-07-01 16:52:20,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:311: done (15 left) 2014-07-01 16:52:20,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:313: done (14 left) 2014-07-01 16:52:20,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: write initial oid 328 2014-07-01 16:52:20,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: seq_num 327 ranges {740256=575251,1775350=400783,2860932=455055,3751925=188331} 2014-07-01 16:52:20,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0437250-328 from 740256 to 1315507 tid 1 2014-07-01 16:52:20,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0437250-328 from 1775350 to 2176133 tid 2 2014-07-01 16:52:20,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0437250-328 from 2860932 to 3315987 tid 3 2014-07-01 16:52:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: writing mira0437250-328 from 3751925 to 3940256 tid 4 2014-07-01 16:52:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: write initial oid 329 2014-07-01 16:52:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: seq_num 328 ranges {434461=583151,1639881=622402,2963980=589397,3634460=1} 2014-07-01 16:52:20,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0437250-329 from 434461 to 1017612 tid 1 2014-07-01 16:52:20,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0437250-329 from 1639881 to 2262283 tid 2 2014-07-01 16:52:20,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0437250-329 from 2963980 to 3553377 tid 3 2014-07-01 16:52:20,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: writing mira0437250-329 from 3634460 to 3634461 tid 4 2014-07-01 16:52:20,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 1 to mira0437250-318 2014-07-01 16:52:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 2 to mira0437250-318 2014-07-01 16:52:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 3 to mira0437250-318 2014-07-01 16:52:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 4 to mira0437250-318 2014-07-01 16:52:20,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 6 to mira0437250-318 2014-07-01 16:52:20,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 1 to mira0437250-320 2014-07-01 16:52:20,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 2 to mira0437250-320 2014-07-01 16:52:20,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 3 to mira0437250-320 2014-07-01 16:52:20,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 1 to mira0437250-322 2014-07-01 16:52:20,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 5 to mira0437250-320 2014-07-01 16:52:20,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 3 to mira0437250-322 2014-07-01 16:52:20,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: finishing write tid 6 to mira0437250-320 2014-07-01 16:52:20,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 67 (ObjNum 319 snap 0 seq_num 319) dirty exists 2014-07-01 16:52:20,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: finishing write tid 4 to mira0437250-322 2014-07-01 16:52:20,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 72 (ObjNum 321 snap 0 seq_num 321) dirty exists 2014-07-01 16:52:20,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 1 to mira0437250-314 2014-07-01 16:52:20,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:320: done (15 left) 2014-07-01 16:52:20,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:322: done (14 left) 2014-07-01 16:52:20,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: write initial oid 330 2014-07-01 16:52:20,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: seq_num 329 ranges {379290=1} 2014-07-01 16:52:20,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: writing mira0437250-330 from 379290 to 379291 tid 1 2014-07-01 16:52:20,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: write initial oid 331 2014-07-01 16:52:20,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: seq_num 330 ranges {440473=526760,1640472=1} 2014-07-01 16:52:20,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: writing mira0437250-331 from 440473 to 967233 tid 1 2014-07-01 16:52:20,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: writing mira0437250-331 from 1640472 to 1640473 tid 2 2014-07-01 16:52:20,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 1 to mira0437250-325 2014-07-01 16:52:20,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 2 to mira0437250-325 2014-07-01 16:52:20,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 4 to mira0437250-325 2014-07-01 16:52:20,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 1 to mira0437250-324 2014-07-01 16:52:20,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 2 to mira0437250-324 2014-07-01 16:52:20,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 3 to mira0437250-324 2014-07-01 16:52:20,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 5 to mira0437250-324 2014-07-01 16:52:20,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: finishing write tid 6 to mira0437250-324 2014-07-01 16:52:20,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 76 (ObjNum 323 snap 0 seq_num 323) dirty exists 2014-07-01 16:52:20,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:324: done (15 left) 2014-07-01 16:52:20,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: write initial oid 332 2014-07-01 16:52:20,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: seq_num 331 ranges {535703=498564,1335702=1} 2014-07-01 16:52:20,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: writing mira0437250-332 from 535703 to 1034267 tid 1 2014-07-01 16:52:20,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: writing mira0437250-332 from 1335702 to 1335703 tid 2 2014-07-01 16:52:20,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: finishing write tid 7 to mira0437250-318 2014-07-01 16:52:20,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 57 (ObjNum 317 snap 0 seq_num 317) dirty exists 2014-07-01 16:52:20,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 1 to mira0437250-316 2014-07-01 16:52:20,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 2 to mira0437250-316 2014-07-01 16:52:20,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:318: done (15 left) 2014-07-01 16:52:20,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: write initial oid 333 2014-07-01 16:52:20,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: seq_num 332 ranges {480532=507489,1608091=472441} 2014-07-01 16:52:20,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: writing mira0437250-333 from 480532 to 988021 tid 1 2014-07-01 16:52:20,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: writing mira0437250-333 from 1608091 to 2080532 tid 2 2014-07-01 16:52:20,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:20,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 4 to mira0437250-316 2014-07-01 16:52:20,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 1 to mira0437250-327 2014-07-01 16:52:20,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 3 to mira0437250-327 2014-07-01 16:52:20,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 1 to mira0437250-317 2014-07-01 16:52:20,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 2 to mira0437250-317 2014-07-01 16:52:20,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 4 to mira0437250-317 2014-07-01 16:52:20,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: finishing write tid 5 to mira0437250-317 2014-07-01 16:52:20,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 63 (ObjNum 316 snap 0 seq_num 316) dirty exists 2014-07-01 16:52:20,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:317: done (15 left) 2014-07-01 16:52:20,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: write initial oid 334 2014-07-01 16:52:20,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: seq_num 333 ranges {424338=479293,1686127=723778,2814013=10325} 2014-07-01 16:52:20,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: writing mira0437250-334 from 424338 to 903631 tid 1 2014-07-01 16:52:20,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: writing mira0437250-334 from 1686127 to 2409905 tid 2 2014-07-01 16:52:20,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: writing mira0437250-334 from 2814013 to 2824338 tid 3 2014-07-01 16:52:20,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 1 to mira0437250-330 2014-07-01 16:52:21,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 3 to mira0437250-330 2014-07-01 16:52:21,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 1 to mira0437250-331 2014-07-01 16:52:21,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 2 to mira0437250-331 2014-07-01 16:52:21,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 4 to mira0437250-331 2014-07-01 16:52:21,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: finishing write tid 5 to mira0437250-331 2014-07-01 16:52:21,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 79 (ObjNum 330 snap 0 seq_num 330) dirty exists 2014-07-01 16:52:21,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:331: done (15 left) 2014-07-01 16:52:21,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: write initial oid 335 2014-07-01 16:52:21,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: seq_num 334 ranges {636944=602521,1667011=429045,2517062=519882} 2014-07-01 16:52:21,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0437250-335 from 636944 to 1239465 tid 1 2014-07-01 16:52:21,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0437250-335 from 1667011 to 2096056 tid 2 2014-07-01 16:52:21,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: writing mira0437250-335 from 2517062 to 3036944 tid 3 2014-07-01 16:52:21,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 1 to mira0437250-332 2014-07-01 16:52:21,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 2 to mira0437250-332 2014-07-01 16:52:21,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 4 to mira0437250-332 2014-07-01 16:52:21,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: finishing write tid 5 to mira0437250-316 2014-07-01 16:52:21,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 53 (ObjNum 315 snap 0 seq_num 315) dirty exists 2014-07-01 16:52:21,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:316: done (15 left) 2014-07-01 16:52:21,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: write initial oid 336 2014-07-01 16:52:21,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: seq_num 335 ranges {581774=575350,1595672=499240,2816464=669763,3781773=1} 2014-07-01 16:52:21,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0437250-336 from 581774 to 1157124 tid 1 2014-07-01 16:52:21,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0437250-336 from 1595672 to 2094912 tid 2 2014-07-01 16:52:21,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0437250-336 from 2816464 to 3486227 tid 3 2014-07-01 16:52:21,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: writing mira0437250-336 from 3781773 to 3781774 tid 4 2014-07-01 16:52:21,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: finishing write tid 5 to mira0437250-332 2014-07-01 16:52:21,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 70 (ObjNum 331 snap 0 seq_num 331) dirty exists 2014-07-01 16:52:21,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:332: done (15 left) 2014-07-01 16:52:21,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: write initial oid 337 2014-07-01 16:52:21,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: seq_num 336 ranges {524554=1} 2014-07-01 16:52:21,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: writing mira0437250-337 from 524554 to 524555 tid 1 2014-07-01 16:52:21,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 2 to mira0437250-314 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 4 to mira0437250-314 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 1 to mira0437250-315 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 2 to mira0437250-315 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 4 to mira0437250-315 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 1 to mira0437250-319 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 2 to mira0437250-319 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 3 to mira0437250-319 2014-07-01 16:52:21,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 5 to mira0437250-319 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 1 to mira0437250-321 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 3 to mira0437250-321 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 1 to mira0437250-323 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: finishing write tid 5 to mira0437250-314 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 63 (ObjNum 313 snap 0 seq_num 313) dirty exists 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 3 to mira0437250-323 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:314: done (15 left) 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: write initial oid 338 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: seq_num 337 ranges {469385=554030,1269384=1} 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: writing mira0437250-338 from 469385 to 1023415 tid 1 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: writing mira0437250-338 from 1269384 to 1269385 tid 2 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: finishing write tid 4 to mira0437250-323 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 62 (ObjNum 322 snap 0 seq_num 322) dirty exists 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: finishing write tid 5 to mira0437250-325 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 58 (ObjNum 324 snap 0 seq_num 324) dirty exists 2014-07-01 16:52:21,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: finishing write tid 5 to mira0437250-315 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 73 (ObjNum 314 snap 0 seq_num 314) dirty exists 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: finishing write tid 4 to mira0437250-330 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 50 (ObjNum 329 snap 0 seq_num 329) dirty exists 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 1 to mira0437250-326 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 2 to mira0437250-326 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 3 to mira0437250-326 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 5 to mira0437250-326 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 1 to mira0437250-328 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:315: done (15 left) 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:323: done (14 left) 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:325: done (13 left) 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:330: done (12 left) 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: write initial oid 339 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: seq_num 338 ranges {564615=400000} 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: writing mira0437250-339 from 564615 to 964615 tid 1 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: write initial oid 340 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: seq_num 339 ranges {625797=650087,1712232=513565} 2014-07-01 16:52:21,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: writing mira0437250-340 from 625797 to 1275884 tid 1 2014-07-01 16:52:21,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: writing mira0437250-340 from 1712232 to 2225797 tid 2 2014-07-01 16:52:21,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: write initial oid 341 2014-07-01 16:52:21,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: seq_num 340 ranges {570626=505539,1674939=600616,2970625=1} 2014-07-01 16:52:21,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0437250-341 from 570626 to 1076165 tid 1 2014-07-01 16:52:21,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0437250-341 from 1674939 to 2275555 tid 2 2014-07-01 16:52:21,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: writing mira0437250-341 from 2970625 to 2970626 tid 3 2014-07-01 16:52:21,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: write initial oid 342 2014-07-01 16:52:21,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: seq_num 341 ranges {665856=477344,1788048=422235,2652167=13689} 2014-07-01 16:52:21,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: writing mira0437250-342 from 665856 to 1143200 tid 1 2014-07-01 16:52:21,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: writing mira0437250-342 from 1788048 to 2210283 tid 2 2014-07-01 16:52:21,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: writing mira0437250-342 from 2652167 to 2665856 tid 3 2014-07-01 16:52:21,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 2 to mira0437250-328 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 3 to mira0437250-328 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 4 to mira0437250-328 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 6 to mira0437250-328 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 1 to mira0437250-329 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 1 to mira0437250-334 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 2 to mira0437250-334 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 3 to mira0437250-334 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 2 to mira0437250-329 2014-07-01 16:52:21,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 5 to mira0437250-334 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 1 to mira0437250-333 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: finishing write tid 4 to mira0437250-327 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 55 (ObjNum 326 snap 0 seq_num 326) dirty exists 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 2 to mira0437250-333 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 3 to mira0437250-329 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:327: done (15 left) 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: write initial oid 343 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: seq_num 342 ranges {762110=601596,1902180=459930} 2014-07-01 16:52:21,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: writing mira0437250-343 from 762110 to 1363706 tid 1 2014-07-01 16:52:21,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: writing mira0437250-343 from 1902180 to 2362110 tid 2 2014-07-01 16:52:21,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 4 to mira0437250-333 2014-07-01 16:52:21,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 1 to mira0437250-335 2014-07-01 16:52:21,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 2 to mira0437250-335 2014-07-01 16:52:21,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 3 to mira0437250-335 2014-07-01 16:52:21,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 5 to mira0437250-335 2014-07-01 16:52:21,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 4 to mira0437250-329 2014-07-01 16:52:21,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 6 to mira0437250-329 2014-07-01 16:52:21,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: finishing write tid 6 to mira0437250-319 2014-07-01 16:52:21,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 67 (ObjNum 318 snap 0 seq_num 318) dirty exists 2014-07-01 16:52:21,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: finishing write tid 4 to mira0437250-321 2014-07-01 16:52:21,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 75 (ObjNum 320 snap 0 seq_num 320) dirty exists 2014-07-01 16:52:21,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: finishing write tid 5 to mira0437250-333 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 65 (ObjNum 332 snap 0 seq_num 332) dirty exists 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:319: done (15 left) 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:321: done (14 left) 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:333: done (13 left) 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: write initial oid 344 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: seq_num 343 ranges {670842=1} 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: writing mira0437250-344 from 670842 to 670843 tid 1 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: write initial oid 345 2014-07-01 16:52:21,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: seq_num 344 ranges {367096=1} 2014-07-01 16:52:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: writing mira0437250-345 from 367096 to 367097 tid 1 2014-07-01 16:52:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: write initial oid 346 2014-07-01 16:52:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: seq_num 345 ranges {62326=1} 2014-07-01 16:52:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: writing mira0437250-346 from 62326 to 62327 tid 1 2014-07-01 16:52:21,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 1 to mira0437250-337 2014-07-01 16:52:21,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: finishing write tid 6 to mira0437250-335 2014-07-01 16:52:21,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 69 (ObjNum 334 snap 0 seq_num 334) dirty exists 2014-07-01 16:52:21,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:335: done (15 left) 2014-07-01 16:52:21,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: write initial oid 347 2014-07-01 16:52:21,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: seq_num 346 ranges {654709=676334,1735094=119615} 2014-07-01 16:52:21,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: writing mira0437250-347 from 654709 to 1331043 tid 1 2014-07-01 16:52:21,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: writing mira0437250-347 from 1735094 to 1854709 tid 2 2014-07-01 16:52:21,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:21,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: finishing write tid 6 to mira0437250-326 2014-07-01 16:52:21,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 71 (ObjNum 325 snap 0 seq_num 325) dirty exists 2014-07-01 16:52:21,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: finishing write tid 7 to mira0437250-329 2014-07-01 16:52:21,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 329 v 85 (ObjNum 328 snap 0 seq_num 328) dirty exists 2014-07-01 16:52:21,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: finishing write tid 7 to mira0437250-328 2014-07-01 16:52:21,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 80 (ObjNum 327 snap 0 seq_num 327) dirty exists 2014-07-01 16:52:21,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 1 to mira0437250-339 2014-07-01 16:52:21,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:326: done (15 left) 2014-07-01 16:52:21,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:328: done (14 left) 2014-07-01 16:52:21,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:329: done (13 left) 2014-07-01 16:52:21,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: write initial oid 348 2014-07-01 16:52:21,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: seq_num 347 ranges {750962=648138,1550961=1} 2014-07-01 16:52:21,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: writing mira0437250-348 from 750962 to 1399100 tid 1 2014-07-01 16:52:21,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: writing mira0437250-348 from 1550961 to 1550962 tid 2 2014-07-01 16:52:21,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: write initial oid 349 2014-07-01 16:52:21,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: seq_num 348 ranges {562544=655014,1678685=83859} 2014-07-01 16:52:21,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: finishing write tid 1 to mira0435601-322 2014-07-01 16:52:21,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: finishing write tid 3 to mira0435601-322 2014-07-01 16:52:21,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: finishing write tid 4 to mira0435601-322 2014-07-01 16:52:21,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 254 (ObjNum 1560 snap 0 seq_num 1560) dirty exists 2014-07-01 16:52:21,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: expect (ObjNum 1322 snap 0 seq_num 1322) 2014-07-01 16:52:21,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: writing mira0437250-349 from 562544 to 1217558 tid 1 2014-07-01 16:52:22,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: writing mira0437250-349 from 1678685 to 1762544 tid 2 2014-07-01 16:52:22,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: write initial oid 350 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: seq_num 349 ranges {755950=627843,1889970=521717,2856276=699674} 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0437250-350 from 755950 to 1383793 tid 1 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0437250-350 from 1889970 to 2411687 tid 2 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: writing mira0437250-350 from 2856276 to 3555950 tid 3 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:22,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 3 to mira0437250-339 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 1 to mira0437250-340 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 2 to mira0437250-340 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 4 to mira0437250-340 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 1 to mira0437250-342 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 3 to mira0437250-337 2014-07-01 16:52:22,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 2 to mira0437250-342 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 3 to mira0437250-342 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 5 to mira0437250-342 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: finishing write tid 4 to mira0437250-337 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 81 (ObjNum 336 snap 0 seq_num 336) dirty exists 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 1 to mira0437250-345 2014-07-01 16:52:22,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 3 to mira0437250-345 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: finishing write tid 4 to mira0437250-345 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 345 v 72 (ObjNum 344 snap 0 seq_num 344) dirty exists 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 1 to mira0437250-336 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:337: done (15 left) 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:345: done (14 left) 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: write initial oid 351 2014-07-01 16:52:22,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: seq_num 350 ranges {452204=751071,1871878=591912,3208925=43279} 2014-07-01 16:52:22,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: writing mira0437250-351 from 452204 to 1203275 tid 1 2014-07-01 16:52:22,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: writing mira0437250-351 from 1871878 to 2463790 tid 2 2014-07-01 16:52:22,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: writing mira0437250-351 from 3208925 to 3252204 tid 3 2014-07-01 16:52:22,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: write initial oid 352 2014-07-01 16:52:22,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: seq_num 351 ranges {547433=571451,1682137=662107,2947432=1} 2014-07-01 16:52:22,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0437250-352 from 547433 to 1118884 tid 1 2014-07-01 16:52:22,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0437250-352 from 1682137 to 2344244 tid 2 2014-07-01 16:52:22,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: writing mira0437250-352 from 2947432 to 2947433 tid 3 2014-07-01 16:52:22,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:22,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: finishing write tid 4 to mira0437250-339 2014-07-01 16:52:22,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 59 (ObjNum 338 snap 0 seq_num 338) dirty exists 2014-07-01 16:52:22,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 1 to mira0437250-338 2014-07-01 16:52:22,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:339: done (15 left) 2014-07-01 16:52:22,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: write initial oid 353 2014-07-01 16:52:22,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: seq_num 352 ranges {457191=578328,1257190=1} 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: writing mira0437250-353 from 457191 to 1035519 tid 1 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: writing mira0437250-353 from 1257190 to 1257191 tid 2 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 2 to mira0437250-338 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 4 to mira0437250-338 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 2 to mira0437250-336 2014-07-01 16:52:22,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 1 to mira0437250-341 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 2 to mira0437250-341 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 3 to mira0437250-341 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 3 to mira0437250-336 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 5 to mira0437250-341 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 1 to mira0437250-343 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 4 to mira0437250-336 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 2 to mira0437250-343 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 4 to mira0437250-343 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 6 to mira0437250-336 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 1 to mira0437250-346 2014-07-01 16:52:22,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: finishing write tid 7 to mira0437250-336 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 90 (ObjNum 335 snap 0 seq_num 335) dirty exists 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 1 to mira0437250-344 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 3 to mira0437250-346 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 3 to mira0437250-344 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: finishing write tid 5 to mira0437250-340 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 52 (ObjNum 339 snap 0 seq_num 339) dirty exists 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: finishing write tid 6 to mira0437250-334 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 75 (ObjNum 333 snap 0 seq_num 333) dirty exists 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: finishing write tid 6 to mira0437250-342 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 56 (ObjNum 341 snap 0 seq_num 341) dirty exists 2014-07-01 16:52:22,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:334: done (15 left) 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:336: done (14 left) 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:340: done (13 left) 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:342: done (12 left) 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: write initial oid 354 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: seq_num 353 ranges {552421=400000} 2014-07-01 16:52:22,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: writing mira0437250-354 from 552421 to 952421 tid 1 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: write initial oid 355 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: seq_num 354 ranges {647650=1} 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: writing mira0437250-355 from 647650 to 647651 tid 1 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: write initial oid 356 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: seq_num 355 ranges {592480=646189,1392479=1} 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: writing mira0437250-356 from 592480 to 1238669 tid 1 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: writing mira0437250-356 from 1392479 to 1392480 tid 2 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: write initial oid 357 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: seq_num 356 ranges {536286=654089,1665278=471008} 2014-07-01 16:52:22,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: writing mira0437250-357 from 536286 to 1190375 tid 1 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: writing mira0437250-357 from 1665278 to 2136286 tid 2 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: finishing write tid 4 to mira0437250-344 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 71 (ObjNum 343 snap 0 seq_num 343) dirty exists 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 1 to mira0437250-347 2014-07-01 16:52:22,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: finishing write tid 5 to mira0437250-338 2014-07-01 16:52:22,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 83 (ObjNum 337 snap 0 seq_num 337) dirty exists 2014-07-01 16:52:22,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: finishing write tid 4 to mira0437250-346 2014-07-01 16:52:22,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 72 (ObjNum 345 snap 0 seq_num 345) dirty exists 2014-07-01 16:52:22,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 2 to mira0437250-347 2014-07-01 16:52:22,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 4 to mira0437250-347 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: finishing write tid 5 to mira0437250-347 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 84 (ObjNum 346 snap 0 seq_num 346) dirty exists 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: finishing write tid 6 to mira0437250-341 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 87 (ObjNum 340 snap 0 seq_num 340) dirty exists 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: finishing write tid 5 to mira0437250-343 2014-07-01 16:52:22,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 90 (ObjNum 342 snap 0 seq_num 342) dirty exists 2014-07-01 16:52:22,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 1 to mira0437250-348 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 2 to mira0437250-348 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 4 to mira0437250-348 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: finishing write tid 5 to mira0437250-348 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 73 (ObjNum 347 snap 0 seq_num 347) dirty exists 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 1 to mira0437250-349 2014-07-01 16:52:22,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 2 to mira0437250-349 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 4 to mira0437250-349 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: finishing write tid 5 to mira0437250-349 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 66 (ObjNum 348 snap 0 seq_num 348) dirty exists 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 1 to mira0437250-350 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 2 to mira0437250-350 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 3 to mira0437250-350 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 5 to mira0437250-350 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: finishing write tid 6 to mira0437250-350 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 76 (ObjNum 349 snap 0 seq_num 349) dirty exists 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 1 to mira0437250-351 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 2 to mira0437250-351 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 3 to mira0437250-351 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 5 to mira0437250-351 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: finishing write tid 6 to mira0437250-351 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 63 (ObjNum 350 snap 0 seq_num 350) dirty exists 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 1 to mira0437250-352 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 2 to mira0437250-352 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 3 to mira0437250-352 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 5 to mira0437250-352 2014-07-01 16:52:22,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: finishing write tid 6 to mira0437250-352 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 352 v 98 (ObjNum 351 snap 0 seq_num 351) dirty exists 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 1 to mira0437250-353 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 2 to mira0437250-353 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 4 to mira0437250-353 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: finishing write tid 5 to mira0437250-353 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 66 (ObjNum 352 snap 0 seq_num 352) dirty exists 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 1 to mira0437250-354 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 1 to mira0437250-355 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 3 to mira0437250-354 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: finishing write tid 4 to mira0437250-354 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 354 v 92 (ObjNum 353 snap 0 seq_num 353) dirty exists 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 3 to mira0437250-355 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: finishing write tid 4 to mira0437250-355 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 68 (ObjNum 354 snap 0 seq_num 354) dirty exists 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 1 to mira0437250-356 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 2 to mira0437250-356 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 4 to mira0437250-356 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: finishing write tid 5 to mira0437250-356 2014-07-01 16:52:22,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 356 v 95 (ObjNum 355 snap 0 seq_num 355) dirty exists 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 1 to mira0437250-357 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:338: done (15 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:341: done (14 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:343: done (13 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:344: done (12 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:346: done (11 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:347: done (10 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:348: done (9 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:349: done (8 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:350: done (7 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:351: done (6 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:352: done (5 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:353: done (4 left) 2014-07-01 16:52:22,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:354: done (3 left) 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:355: done (2 left) 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:356: done (1 left) 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: write initial oid 358 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: seq_num 357 ranges {748892=625893,1859666=468750,2348891=1} 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: writing mira0437250-358 from 748892 to 1374785 tid 1 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: writing mira0437250-358 from 1859666 to 2328416 tid 2 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: writing mira0437250-358 from 2348891 to 2348892 tid 3 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: write initial oid 359 2014-07-01 16:52:22,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: seq_num 358 ranges {693722=749122,1973799=690370,3093721=1} 2014-07-01 16:52:22,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0437250-359 from 693722 to 1442844 tid 1 2014-07-01 16:52:22,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0437250-359 from 1973799 to 2664169 tid 2 2014-07-01 16:52:22,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: writing mira0437250-359 from 3093721 to 3093722 tid 3 2014-07-01 16:52:22,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: write initial oid 360 2014-07-01 16:52:22,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: seq_num 359 ranges {637527=720926,1935482=760565,3460784=376743} 2014-07-01 16:52:22,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0437250-360 from 637527 to 1358453 tid 1 2014-07-01 16:52:22,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0437250-360 from 1935482 to 2696047 tid 2 2014-07-01 16:52:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: writing mira0437250-360 from 3460784 to 3837527 tid 3 2014-07-01 16:52:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: write initial oid 361 2014-07-01 16:52:22,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: seq_num 360 ranges {733781=729850,2051662=430760,2997305=478886,3533780=1} 2014-07-01 16:52:22,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0437250-361 from 733781 to 1463631 tid 1 2014-07-01 16:52:22,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0437250-361 from 2051662 to 2482422 tid 2 2014-07-01 16:52:22,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0437250-361 from 2997305 to 3476191 tid 3 2014-07-01 16:52:22,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: writing mira0437250-361 from 3533780 to 3533781 tid 4 2014-07-01 16:52:22,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: write initial oid 362 2014-07-01 16:52:22,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: seq_num 361 ranges {792914=1} 2014-07-01 16:52:22,516.516 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: writing mira0437250-362 from 792914 to 792915 tid 1 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: write initial oid 363 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: seq_num 362 ranges {737744=673459,1537743=1} 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: writing mira0437250-363 from 737744 to 1411203 tid 1 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: writing mira0437250-363 from 1537743 to 1537744 tid 2 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: write initial oid 364 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: seq_num 363 ranges {433998=646287,1233997=1} 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: writing mira0437250-364 from 433998 to 1080285 tid 1 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: writing mira0437250-364 from 1233997 to 1233998 tid 2 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: write initial oid 365 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: seq_num 364 ranges {777804=653164,1977803=1} 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: writing mira0437250-365 from 777804 to 1430968 tid 1 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: writing mira0437250-365 from 1977803 to 1977804 tid 2 2014-07-01 16:52:22,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: write initial oid 366 2014-07-01 16:52:22,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: seq_num 365 ranges {722634=624968,1846260=568232,2722633=1} 2014-07-01 16:52:22,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: writing mira0437250-366 from 722634 to 1347602 tid 1 2014-07-01 16:52:22,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: writing mira0437250-366 from 1846260 to 2414492 tid 2 2014-07-01 16:52:22,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: writing mira0437250-366 from 2722633 to 2722634 tid 3 2014-07-01 16:52:22,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: write initial oid 367 2014-07-01 16:52:22,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: seq_num 366 ranges {534215=748196,2075719=789851,2934214=1} 2014-07-01 16:52:22,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0437250-367 from 534215 to 1282411 tid 1 2014-07-01 16:52:22,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0437250-367 from 2075719 to 2865570 tid 2 2014-07-01 16:52:22,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: writing mira0437250-367 from 2934214 to 2934215 tid 3 2014-07-01 16:52:22,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: write initial oid 368 2014-07-01 16:52:22,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: seq_num 367 ranges {479045=721025,1755804=460046,2866940=407828,3679044=1} 2014-07-01 16:52:22,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0437250-368 from 479045 to 1200070 tid 1 2014-07-01 16:52:22,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0437250-368 from 1755804 to 2215850 tid 2 2014-07-01 16:52:22,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0437250-368 from 2866940 to 3274768 tid 3 2014-07-01 16:52:22,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: writing mira0437250-368 from 3679044 to 3679045 tid 4 2014-07-01 16:52:22,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: write initial oid 369 2014-07-01 16:52:22,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: seq_num 368 ranges {575299=727901,1905008=530242,3103238=272061} 2014-07-01 16:52:22,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: writing mira0437250-369 from 575299 to 1303200 tid 1 2014-07-01 16:52:22,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: writing mira0437250-369 from 1905008 to 2435250 tid 2 2014-07-01 16:52:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: writing mira0437250-369 from 3103238 to 3375299 tid 3 2014-07-01 16:52:22,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: write initial oid 370 2014-07-01 16:52:22,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: seq_num 369 ranges {118080=1} 2014-07-01 16:52:22,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: writing mira0437250-370 from 118080 to 118081 tid 1 2014-07-01 16:52:22,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: write initial oid 371 2014-07-01 16:52:22,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: seq_num 370 ranges {579262=422934,1379261=1} 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: writing mira0437250-371 from 579262 to 1002196 tid 1 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: writing mira0437250-371 from 1379261 to 1379262 tid 2 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: write initial oid 372 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: seq_num 371 ranges {674492=400000} 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: writing mira0437250-372 from 674492 to 1074492 tid 1 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 2 to mira0437250-357 2014-07-01 16:52:22,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 4 to mira0437250-357 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: finishing write tid 5 to mira0437250-357 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 78 (ObjNum 356 snap 0 seq_num 356) dirty exists 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 1 to mira0437250-358 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 2 to mira0437250-358 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 3 to mira0437250-358 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 5 to mira0437250-358 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: finishing write tid 6 to mira0437250-358 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 77 (ObjNum 357 snap 0 seq_num 357) dirty exists 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 1 to mira0437250-359 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 2 to mira0437250-359 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 3 to mira0437250-359 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 5 to mira0437250-359 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: finishing write tid 6 to mira0437250-359 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 59 (ObjNum 358 snap 0 seq_num 358) dirty exists 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 1 to mira0437250-360 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 2 to mira0437250-360 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 3 to mira0437250-360 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 5 to mira0437250-360 2014-07-01 16:52:22,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: finishing write tid 6 to mira0437250-360 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 82 (ObjNum 359 snap 0 seq_num 359) dirty exists 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 1 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 2 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 3 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 4 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 6 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 1 to mira0437250-362 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: finishing write tid 7 to mira0437250-361 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 361 v 100 (ObjNum 360 snap 0 seq_num 360) dirty exists 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 3 to mira0437250-362 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: finishing write tid 4 to mira0437250-362 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 102 (ObjNum 361 snap 0 seq_num 361) dirty exists 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 1 to mira0437250-363 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 2 to mira0437250-363 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 4 to mira0437250-363 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: finishing write tid 5 to mira0437250-363 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 61 (ObjNum 362 snap 0 seq_num 362) dirty exists 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 1 to mira0437250-364 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 2 to mira0437250-364 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 4 to mira0437250-364 2014-07-01 16:52:22,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: finishing write tid 5 to mira0437250-364 2014-07-01 16:52:22,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 364 v 93 (ObjNum 363 snap 0 seq_num 363) dirty exists 2014-07-01 16:52:22,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 1 to mira0437250-365 2014-07-01 16:52:22,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 2 to mira0437250-365 2014-07-01 16:52:22,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 4 to mira0437250-365 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: finishing write tid 5 to mira0437250-365 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 365 v 64 (ObjNum 364 snap 0 seq_num 364) dirty exists 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 1 to mira0437250-366 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 2 to mira0437250-366 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 3 to mira0437250-366 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 5 to mira0437250-366 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: finishing write tid 6 to mira0437250-366 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 54 (ObjNum 365 snap 0 seq_num 365) dirty exists 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 1 to mira0437250-367 2014-07-01 16:52:22,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 2 to mira0437250-367 2014-07-01 16:52:22,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 3 to mira0437250-367 2014-07-01 16:52:22,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 5 to mira0437250-367 2014-07-01 16:52:22,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: finishing write tid 6 to mira0437250-367 2014-07-01 16:52:22,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 70 (ObjNum 366 snap 0 seq_num 366) dirty exists 2014-07-01 16:52:22,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 1 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 2 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 3 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 4 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 6 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: finishing write tid 7 to mira0437250-368 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 107 (ObjNum 367 snap 0 seq_num 367) dirty exists 2014-07-01 16:52:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 1 to mira0437250-369 2014-07-01 16:52:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 2 to mira0437250-369 2014-07-01 16:52:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 3 to mira0437250-369 2014-07-01 16:52:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 1 to mira0437250-370 2014-07-01 16:52:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 5 to mira0437250-369 2014-07-01 16:52:22,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 3 to mira0437250-370 2014-07-01 16:52:22,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: finishing write tid 4 to mira0437250-370 2014-07-01 16:52:22,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 66 (ObjNum 369 snap 0 seq_num 369) dirty exists 2014-07-01 16:52:22,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: finishing write tid 6 to mira0437250-369 2014-07-01 16:52:22,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 369 v 86 (ObjNum 368 snap 0 seq_num 368) dirty exists 2014-07-01 16:52:22,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 1 to mira0437250-371 2014-07-01 16:52:22,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 2 to mira0437250-371 2014-07-01 16:52:22,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 4 to mira0437250-371 2014-07-01 16:52:22,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: finishing write tid 5 to mira0437250-371 2014-07-01 16:52:22,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 80 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 16:52:22,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:357: done (15 left) 2014-07-01 16:52:22,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:358: done (14 left) 2014-07-01 16:52:22,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:359: done (13 left) 2014-07-01 16:52:22,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:360: done (12 left) 2014-07-01 16:52:22,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:361: done (11 left) 2014-07-01 16:52:22,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:362: done (10 left) 2014-07-01 16:52:22,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:363: done (9 left) 2014-07-01 16:52:22,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:364: done (8 left) 2014-07-01 16:52:22,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:365: done (7 left) 2014-07-01 16:52:22,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:366: done (6 left) 2014-07-01 16:52:22,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:367: done (5 left) 2014-07-01 16:52:22,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:368: done (4 left) 2014-07-01 16:52:22,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:369: done (3 left) 2014-07-01 16:52:22,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:370: done (2 left) 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:371: done (1 left) 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: write initial oid 373 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: seq_num 372 ranges {619322=402638,1586496=232826} 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: writing mira0437250-373 from 619322 to 1021960 tid 1 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: writing mira0437250-373 from 1586496 to 1819322 tid 2 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: write initial oid 374 2014-07-01 16:52:22,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: seq_num 373 ranges {564152=623019,1796757=666689,2564151=1} 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: writing mira0437250-374 from 564152 to 1187171 tid 1 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: writing mira0437250-374 from 1796757 to 2463446 tid 2 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: writing mira0437250-374 from 2564151 to 2564152 tid 3 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: write initial oid 375 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: seq_num 374 ranges {659381=747271,2062312=197069} 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: writing mira0437250-375 from 659381 to 1406652 tid 1 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: writing mira0437250-375 from 2062312 to 2259381 tid 2 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: write initial oid 376 2014-07-01 16:52:22,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: seq_num 375 ranges {720563=719075,1857724=558504,3105094=415469} 2014-07-01 16:52:22,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0437250-376 from 720563 to 1439638 tid 1 2014-07-01 16:52:22,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0437250-376 from 1857724 to 2416228 tid 2 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: writing mira0437250-376 from 3105094 to 3520563 tid 3 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: write initial oid 377 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: seq_num 376 ranges {264368=1} 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: writing mira0437250-377 from 264368 to 264369 tid 1 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: write initial oid 378 2014-07-01 16:52:22,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: seq_num 377 ranges {759599=697756,1932517=698894,3355097=604502} 2014-07-01 16:52:22,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: writing mira0437250-378 from 759599 to 1457355 tid 1 2014-07-01 16:52:22,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: writing mira0437250-378 from 1932517 to 2631411 tid 2 2014-07-01 16:52:22,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: writing mira0437250-378 from 3355097 to 3959599 tid 3 2014-07-01 16:52:23,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: write initial oid 379 2014-07-01 16:52:23,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: seq_num 378 ranges {704427=1} 2014-07-01 16:52:23,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: writing mira0437250-379 from 704427 to 704428 tid 1 2014-07-01 16:52:23,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: write initial oid 380 2014-07-01 16:52:23,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: seq_num 379 ranges {764586=793813,1964585=1} 2014-07-01 16:52:23,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: writing mira0437250-380 from 764586 to 1558399 tid 1 2014-07-01 16:52:23,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: writing mira0437250-380 from 1964585 to 1964586 tid 2 2014-07-01 16:52:23,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: write initial oid 381 2014-07-01 16:52:23,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: seq_num 380 ranges {460840=517041,1556194=104646} 2014-07-01 16:52:23,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: writing mira0437250-381 from 460840 to 977881 tid 1 2014-07-01 16:52:23,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: writing mira0437250-381 from 1556194 to 1660840 tid 2 2014-07-01 16:52:23,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: write initial oid 382 2014-07-01 16:52:23,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: seq_num 381 ranges {404646=773518,1801527=603119} 2014-07-01 16:52:23,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: writing mira0437250-382 from 404646 to 1178164 tid 1 2014-07-01 16:52:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: writing mira0437250-382 from 1801527 to 2404646 tid 2 2014-07-01 16:52:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: write initial oid 383 2014-07-01 16:52:23,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: seq_num 382 ranges {749475=496746,1880586=436366,2991625=157850} 2014-07-01 16:52:23,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0437250-383 from 749475 to 1246221 tid 1 2014-07-01 16:52:23,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0437250-383 from 1880586 to 2316952 tid 2 2014-07-01 16:52:23,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: writing mira0437250-383 from 2991625 to 3149475 tid 3 2014-07-01 16:52:23,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: write initial oid 384 2014-07-01 16:52:23,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: seq_num 383 ranges {445729=717126,1843294=657985,2845728=1} 2014-07-01 16:52:23,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0437250-384 from 445729 to 1162855 tid 1 2014-07-01 16:52:23,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0437250-384 from 1843294 to 2501279 tid 2 2014-07-01 16:52:23,204.204 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: writing mira0437250-384 from 2845728 to 2845729 tid 3 2014-07-01 16:52:23,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: write initial oid 385 2014-07-01 16:52:23,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: seq_num 384 ranges {105886=1} 2014-07-01 16:52:23,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: writing mira0437250-385 from 105886 to 105887 tid 1 2014-07-01 16:52:23,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: write initial oid 386 2014-07-01 16:52:23,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: seq_num 385 ranges {449692=400000} 2014-07-01 16:52:23,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: writing mira0437250-386 from 449692 to 849692 tid 1 2014-07-01 16:52:23,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: write initial oid 387 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: seq_num 386 ranges {544921=1} 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: writing mira0437250-387 from 544921 to 544922 tid 1 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 1 to mira0437250-372 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 3 to mira0437250-372 2014-07-01 16:52:23,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: finishing write tid 4 to mira0437250-372 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 61 (ObjNum 371 snap 0 seq_num 371) dirty exists 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 1 to mira0437250-373 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 2 to mira0437250-373 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 4 to mira0437250-373 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: finishing write tid 5 to mira0437250-373 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 64 (ObjNum 372 snap 0 seq_num 372) dirty exists 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 1 to mira0437250-374 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 2 to mira0437250-374 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 3 to mira0437250-374 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 5 to mira0437250-374 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: finishing write tid 6 to mira0437250-374 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 84 (ObjNum 373 snap 0 seq_num 373) dirty exists 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 1 to mira0437250-375 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 2 to mira0437250-375 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 4 to mira0437250-375 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: finishing write tid 5 to mira0437250-375 2014-07-01 16:52:23,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 375 v 69 (ObjNum 374 snap 0 seq_num 374) dirty exists 2014-07-01 16:52:23,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 1 to mira0437250-376 2014-07-01 16:52:23,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 2 to mira0437250-376 2014-07-01 16:52:23,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 1 to mira0437250-377 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 3 to mira0437250-377 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: finishing write tid 4 to mira0437250-377 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 72 (ObjNum 376 snap 0 seq_num 376) dirty exists 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 3 to mira0437250-376 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 5 to mira0437250-376 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: finishing write tid 6 to mira0437250-376 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 102 (ObjNum 375 snap 0 seq_num 375) dirty exists 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 1 to mira0437250-378 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 2 to mira0437250-378 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 1 to mira0437250-379 2014-07-01 16:52:23,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 3 to mira0437250-379 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 3 to mira0437250-378 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: finishing write tid 4 to mira0437250-379 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 70 (ObjNum 378 snap 0 seq_num 378) dirty exists 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 5 to mira0437250-378 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: finishing write tid 6 to mira0437250-378 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 378 v 76 (ObjNum 377 snap 0 seq_num 377) dirty exists 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 1 to mira0437250-380 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 2 to mira0437250-380 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 4 to mira0437250-380 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: finishing write tid 5 to mira0437250-380 2014-07-01 16:52:23,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 380 v 67 (ObjNum 379 snap 0 seq_num 379) dirty exists 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 1 to mira0437250-381 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 1 to mira0437250-382 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:372: done (15 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:373: done (14 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:374: done (13 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:375: done (12 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:376: done (11 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:377: done (10 left) 2014-07-01 16:52:23,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:378: done (9 left) 2014-07-01 16:52:23,285.285 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:52:23,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:379: done (8 left) 2014-07-01 16:52:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:380: done (7 left) 2014-07-01 16:52:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: write initial oid 388 2014-07-01 16:52:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: seq_num 387 ranges {489752=791864,1289751=1} 2014-07-01 16:52:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: writing mira0437250-388 from 489752 to 1281616 tid 1 2014-07-01 16:52:23,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: writing mira0437250-388 from 1289751 to 1289752 tid 2 2014-07-01 16:52:23,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: write initial oid 389 2014-07-01 16:52:23,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: seq_num 388 ranges {550934=516116,1506691=644033,2550933=1} 2014-07-01 16:52:23,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0437250-389 from 550934 to 1067050 tid 1 2014-07-01 16:52:23,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0437250-389 from 1506691 to 2150724 tid 2 2014-07-01 16:52:23,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: writing mira0437250-389 from 2550933 to 2550934 tid 3 2014-07-01 16:52:23,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: write initial oid 390 2014-07-01 16:52:23,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: seq_num 389 ranges {646163=772592,1904470=341693} 2014-07-01 16:52:23,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: writing mira0437250-390 from 646163 to 1418755 tid 1 2014-07-01 16:52:23,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: writing mira0437250-390 from 1904470 to 2246163 tid 2 2014-07-01 16:52:23,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: write initial oid 391 2014-07-01 16:52:23,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: seq_num 390 ranges {590993=495821,1734955=535848,2983252=7741} 2014-07-01 16:52:23,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0437250-391 from 590993 to 1086814 tid 1 2014-07-01 16:52:23,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0437250-391 from 1734955 to 2270803 tid 2 2014-07-01 16:52:23,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: writing mira0437250-391 from 2983252 to 2990993 tid 3 2014-07-01 16:52:23,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: write initial oid 392 2014-07-01 16:52:23,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: seq_num 391 ranges {686223=467625,1696639=606043,2686222=1} 2014-07-01 16:52:23,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: writing mira0437250-392 from 686223 to 1153848 tid 1 2014-07-01 16:52:23,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: writing mira0437250-392 from 1696639 to 2302682 tid 2 2014-07-01 16:52:23,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: writing mira0437250-392 from 2686222 to 2686223 tid 3 2014-07-01 16:52:23,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: write initial oid 393 2014-07-01 16:52:23,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: seq_num 392 ranges {631053=591877,1928147=675214,3198182=232871} 2014-07-01 16:52:23,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: writing mira0437250-393 from 631053 to 1222930 tid 1 2014-07-01 16:52:23,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: writing mira0437250-393 from 1928147 to 2603361 tid 2 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: writing mira0437250-393 from 3198182 to 3431053 tid 3 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: write initial oid 394 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: seq_num 393 ranges {691209=1} 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: writing mira0437250-394 from 691209 to 691210 tid 1 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: write initial oid 395 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: seq_num 394 ranges {386439=1} 2014-07-01 16:52:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: writing mira0437250-395 from 386439 to 386440 tid 1 2014-07-01 16:52:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: write initial oid 396 2014-07-01 16:52:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: seq_num 395 ranges {731270=400000} 2014-07-01 16:52:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: writing mira0437250-396 from 731270 to 1131270 tid 1 2014-07-01 16:52:23,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:23,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 2 to mira0437250-382 2014-07-01 16:52:23,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 4 to mira0437250-382 2014-07-01 16:52:23,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 1 to mira0437250-383 2014-07-01 16:52:23,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 2 to mira0437250-383 2014-07-01 16:52:23,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 3 to mira0437250-383 2014-07-01 16:52:23,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 5 to mira0437250-383 2014-07-01 16:52:23,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 1 to mira0437250-388 2014-07-01 16:52:23,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 2 to mira0437250-388 2014-07-01 16:52:23,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 4 to mira0437250-388 2014-07-01 16:52:23,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 1 to mira0437250-387 2014-07-01 16:52:23,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 3 to mira0437250-387 2014-07-01 16:52:23,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 1 to mira0437250-386 2014-07-01 16:52:23,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: finishing write tid 5 to mira0437250-382 2014-07-01 16:52:23,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 89 (ObjNum 381 snap 0 seq_num 381) dirty exists 2014-07-01 16:52:23,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:382: done (15 left) 2014-07-01 16:52:23,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: write initial oid 397 2014-07-01 16:52:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: seq_num 396 ranges {675076=515191,1643684=231392} 2014-07-01 16:52:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: writing mira0437250-397 from 675076 to 1190267 tid 1 2014-07-01 16:52:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: writing mira0437250-397 from 1643684 to 1875076 tid 2 2014-07-01 16:52:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 3 to mira0437250-386 2014-07-01 16:52:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 1 to mira0437250-390 2014-07-01 16:52:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 2 to mira0437250-390 2014-07-01 16:52:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: finishing write tid 4 to mira0437250-386 2014-07-01 16:52:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 74 (ObjNum 385 snap 0 seq_num 385) dirty exists 2014-07-01 16:52:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 2 to mira0437250-381 2014-07-01 16:52:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: finishing write tid 4 to mira0437250-387 2014-07-01 16:52:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 95 (ObjNum 386 snap 0 seq_num 386) dirty exists 2014-07-01 16:52:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 1 to mira0437250-393 2014-07-01 16:52:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 4 to mira0437250-390 2014-07-01 16:52:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 2 to mira0437250-393 2014-07-01 16:52:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 3 to mira0437250-393 2014-07-01 16:52:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:386: done (15 left) 2014-07-01 16:52:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:387: done (14 left) 2014-07-01 16:52:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: write initial oid 398 2014-07-01 16:52:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: seq_num 397 ranges {487681=770643,1722744=364937} 2014-07-01 16:52:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0437250-398 from 487681 to 1258324 tid 1 2014-07-01 16:52:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: writing mira0437250-398 from 1722744 to 2087681 tid 2 2014-07-01 16:52:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: write initial oid 399 2014-07-01 16:52:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: seq_num 398 ranges {432511=493871,1436876=482881,2634910=197601} 2014-07-01 16:52:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0437250-399 from 432511 to 926382 tid 1 2014-07-01 16:52:23,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0437250-399 from 1436876 to 1919757 tid 2 2014-07-01 16:52:23,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: writing mira0437250-399 from 2634910 to 2832511 tid 3 2014-07-01 16:52:23,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:23,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 1 to mira0437250-394 2014-07-01 16:52:23,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 3 to mira0437250-394 2014-07-01 16:52:23,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 5 to mira0437250-393 2014-07-01 16:52:23,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 1 to mira0437250-395 2014-07-01 16:52:23,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 3 to mira0437250-395 2014-07-01 16:52:23,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: finishing write tid 6 to mira0437250-383 2014-07-01 16:52:23,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 71 (ObjNum 382 snap 0 seq_num 382) dirty exists 2014-07-01 16:52:23,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 4 to mira0437250-381 2014-07-01 16:52:23,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 1 to mira0437250-385 2014-07-01 16:52:23,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 3 to mira0437250-385 2014-07-01 16:52:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 1 to mira0437250-389 2014-07-01 16:52:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 2 to mira0437250-389 2014-07-01 16:52:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:383: done (15 left) 2014-07-01 16:52:23,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: write initial oid 400 2014-07-01 16:52:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: seq_num 399 ranges {776317=466700,1648161=704500,2968361=448740,3576316=1} 2014-07-01 16:52:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0437250-400 from 776317 to 1243017 tid 1 2014-07-01 16:52:23,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0437250-400 from 1648161 to 2352661 tid 2 2014-07-01 16:52:23,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0437250-400 from 2968361 to 3417101 tid 3 2014-07-01 16:52:23,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: writing mira0437250-400 from 3576316 to 3576317 tid 4 2014-07-01 16:52:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 3 to mira0437250-389 2014-07-01 16:52:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 5 to mira0437250-389 2014-07-01 16:52:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 1 to mira0437250-392 2014-07-01 16:52:23,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 2 to mira0437250-392 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 1 to mira0437250-384 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 3 to mira0437250-392 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 5 to mira0437250-392 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 1 to mira0437250-396 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 3 to mira0437250-396 2014-07-01 16:52:23,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 2 to mira0437250-384 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 3 to mira0437250-384 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 1 to mira0437250-391 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 5 to mira0437250-384 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: finishing write tid 5 to mira0437250-388 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 74 (ObjNum 387 snap 0 seq_num 387) dirty exists 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 2 to mira0437250-391 2014-07-01 16:52:23,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:388: done (15 left) 2014-07-01 16:52:23,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: write initial oid 401 2014-07-01 16:52:23,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: seq_num 400 ranges {472571=589928,1478645=774696,2734515=538056} 2014-07-01 16:52:24,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: writing mira0437250-401 from 472571 to 1062499 tid 1 2014-07-01 16:52:24,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: writing mira0437250-401 from 1478645 to 2253341 tid 2 2014-07-01 16:52:24,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: writing mira0437250-401 from 2734515 to 3272571 tid 3 2014-07-01 16:52:24,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 3 to mira0437250-391 2014-07-01 16:52:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 5 to mira0437250-391 2014-07-01 16:52:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: finishing write tid 6 to mira0437250-384 2014-07-01 16:52:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 88 (ObjNum 383 snap 0 seq_num 383) dirty exists 2014-07-01 16:52:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: finishing write tid 5 to mira0437250-390 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 390 v 98 (ObjNum 389 snap 0 seq_num 389) dirty exists 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: finishing write tid 6 to mira0437250-391 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 88 (ObjNum 390 snap 0 seq_num 390) dirty exists 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: finishing write tid 4 to mira0437250-394 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 100 (ObjNum 393 snap 0 seq_num 393) dirty exists 2014-07-01 16:52:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: finishing write tid 5 to mira0437250-381 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 57 (ObjNum 380 snap 0 seq_num 380) dirty exists 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: finishing write tid 4 to mira0437250-385 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 385 v 71 (ObjNum 384 snap 0 seq_num 384) dirty exists 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: finishing write tid 6 to mira0437250-393 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 393 v 93 (ObjNum 392 snap 0 seq_num 392) dirty exists 2014-07-01 16:52:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: finishing write tid 4 to mira0437250-395 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 95 (ObjNum 394 snap 0 seq_num 394) dirty exists 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 1 to mira0437250-397 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: finishing write tid 6 to mira0437250-389 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 75 (ObjNum 388 snap 0 seq_num 388) dirty exists 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 2 to mira0437250-397 2014-07-01 16:52:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 4 to mira0437250-397 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: finishing write tid 6 to mira0437250-392 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 392 v 79 (ObjNum 391 snap 0 seq_num 391) dirty exists 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: finishing write tid 5 to mira0437250-397 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 103 (ObjNum 396 snap 0 seq_num 396) dirty exists 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: finishing write tid 4 to mira0437250-396 2014-07-01 16:52:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 73 (ObjNum 395 snap 0 seq_num 395) dirty exists 2014-07-01 16:52:24,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 1 to mira0437250-398 2014-07-01 16:52:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 2 to mira0437250-398 2014-07-01 16:52:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 4 to mira0437250-398 2014-07-01 16:52:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: finishing write tid 5 to mira0437250-398 2014-07-01 16:52:24,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 73 (ObjNum 397 snap 0 seq_num 397) dirty exists 2014-07-01 16:52:24,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 1 to mira0437250-399 2014-07-01 16:52:24,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 2 to mira0437250-399 2014-07-01 16:52:24,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 3 to mira0437250-399 2014-07-01 16:52:24,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 5 to mira0437250-399 2014-07-01 16:52:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: finishing write tid 6 to mira0437250-399 2014-07-01 16:52:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 399 v 65 (ObjNum 398 snap 0 seq_num 398) dirty exists 2014-07-01 16:52:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 1 to mira0437250-400 2014-07-01 16:52:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 2 to mira0437250-400 2014-07-01 16:52:24,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 3 to mira0437250-400 2014-07-01 16:52:24,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 4 to mira0437250-400 2014-07-01 16:52:24,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 6 to mira0437250-400 2014-07-01 16:52:24,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:381: done (15 left) 2014-07-01 16:52:24,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:384: done (14 left) 2014-07-01 16:52:24,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:385: done (13 left) 2014-07-01 16:52:24,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:389: done (12 left) 2014-07-01 16:52:24,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:390: done (11 left) 2014-07-01 16:52:24,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:391: done (10 left) 2014-07-01 16:52:24,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:392: done (9 left) 2014-07-01 16:52:24,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:393: done (8 left) 2014-07-01 16:52:24,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:394: done (7 left) 2014-07-01 16:52:24,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:395: done (6 left) 2014-07-01 16:52:24,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:396: done (5 left) 2014-07-01 16:52:24,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:397: done (4 left) 2014-07-01 16:52:24,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:398: done (3 left) 2014-07-01 16:52:24,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:399: done (2 left) 2014-07-01 16:52:24,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: write initial oid 402 2014-07-01 16:52:24,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: seq_num 401 ranges {16375=1} 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: writing mira0437250-402 from 16375 to 16376 tid 1 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: write initial oid 403 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: seq_num 402 ranges {476534=569633,1276533=1} 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: writing mira0437250-403 from 476534 to 1046167 tid 1 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: writing mira0437250-403 from 1276533 to 1276534 tid 2 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: write initial oid 404 2014-07-01 16:52:24,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: seq_num 403 ranges {572788=400000} 2014-07-01 16:52:24,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: writing mira0437250-404 from 572788 to 972788 tid 1 2014-07-01 16:52:24,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: write initial oid 405 2014-07-01 16:52:24,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: seq_num 404 ranges {668016=1} 2014-07-01 16:52:24,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: writing mira0437250-405 from 668016 to 668017 tid 1 2014-07-01 16:52:24,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: write initial oid 406 2014-07-01 16:52:24,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: seq_num 405 ranges {461423=768694,1956889=477096,2461422=1} 2014-07-01 16:52:24,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: writing mira0437250-406 from 461423 to 1230117 tid 1 2014-07-01 16:52:24,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: writing mira0437250-406 from 1956889 to 2433985 tid 2 2014-07-01 16:52:24,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: writing mira0437250-406 from 2461422 to 2461423 tid 3 2014-07-01 16:52:24,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: write initial oid 407 2014-07-01 16:52:24,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: seq_num 406 ranges {673005=492946,1787373=582363,2673004=1} 2014-07-01 16:52:24,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0437250-407 from 673005 to 1165951 tid 1 2014-07-01 16:52:24,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0437250-407 from 1787373 to 2369736 tid 2 2014-07-01 16:52:24,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: writing mira0437250-407 from 2673004 to 2673005 tid 3 2014-07-01 16:52:24,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: write initial oid 408 2014-07-01 16:52:24,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: seq_num 407 ranges {769259=464750,2017857=351402} 2014-07-01 16:52:24,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: writing mira0437250-408 from 769259 to 1234009 tid 1 2014-07-01 16:52:24,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: writing mira0437250-408 from 2017857 to 2369259 tid 2 2014-07-01 16:52:24,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: write initial oid 409 2014-07-01 16:52:24,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: seq_num 408 ranges {162663=1} 2014-07-01 16:52:24,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: writing mira0437250-409 from 162663 to 162664 tid 1 2014-07-01 16:52:24,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: write initial oid 410 2014-07-01 16:52:24,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: seq_num 409 ranges {657894=444455,1826921=544372,2907141=683558,3857893=1} 2014-07-01 16:52:24,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: writing mira0437250-410 from 657894 to 1102349 tid 1 2014-07-01 16:52:24,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: writing mira0437250-410 from 1826921 to 2371293 tid 2 2014-07-01 16:52:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: writing mira0437250-410 from 2907141 to 3590699 tid 3 2014-07-01 16:52:24,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: writing mira0437250-410 from 3857893 to 3857894 tid 4 2014-07-01 16:52:24,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: write initial oid 411 2014-07-01 16:52:24,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: seq_num 410 ranges {753124=567684,1807806=765991,3127567=417900,3553123=1} 2014-07-01 16:52:24,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0437250-411 from 753124 to 1320808 tid 1 2014-07-01 16:52:24,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0437250-411 from 1807806 to 2573797 tid 2 2014-07-01 16:52:24,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0437250-411 from 3127567 to 3545467 tid 3 2014-07-01 16:52:24,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: writing mira0437250-411 from 3553123 to 3553124 tid 4 2014-07-01 16:52:24,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: write initial oid 412 2014-07-01 16:52:24,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: seq_num 411 ranges {661858=539488,1734418=127440} 2014-07-01 16:52:24,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: writing mira0437250-412 from 661858 to 1201346 tid 1 2014-07-01 16:52:24,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: writing mira0437250-412 from 1734418 to 1861858 tid 2 2014-07-01 16:52:24,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: write initial oid 413 2014-07-01 16:52:24,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: seq_num 412 ranges {758111=662716,1558110=1} 2014-07-01 16:52:24,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: writing mira0437250-413 from 758111 to 1420827 tid 1 2014-07-01 16:52:24,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: writing mira0437250-413 from 1558110 to 1558111 tid 2 2014-07-01 16:52:24,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: write initial oid 414 2014-07-01 16:52:24,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: seq_num 413 ranges {454365=519193,1254364=1} 2014-07-01 16:52:24,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: writing mira0437250-414 from 454365 to 973558 tid 1 2014-07-01 16:52:24,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: writing mira0437250-414 from 1254364 to 1254365 tid 2 2014-07-01 16:52:24,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: write initial oid 415 2014-07-01 16:52:24,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: seq_num 414 ranges {646747=642421,1924367=797172,3046746=1} 2014-07-01 16:52:24,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0437250-415 from 646747 to 1289168 tid 1 2014-07-01 16:52:24,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0437250-415 from 1924367 to 2721539 tid 2 2014-07-01 16:52:24,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: writing mira0437250-415 from 3046746 to 3046747 tid 3 2014-07-01 16:52:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: finishing write tid 7 to mira0437250-400 2014-07-01 16:52:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 81 (ObjNum 399 snap 0 seq_num 399) dirty exists 2014-07-01 16:52:24,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 1 to mira0437250-401 2014-07-01 16:52:24,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 2 to mira0437250-401 2014-07-01 16:52:24,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 1 to mira0437250-402 2014-07-01 16:52:24,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 3 to mira0437250-402 2014-07-01 16:52:24,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:400: done (15 left) 2014-07-01 16:52:24,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: write initial oid 416 2014-07-01 16:52:24,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: seq_num 415 ranges {459353=614225,1871203=502439,3029822=229531} 2014-07-01 16:52:24,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0437250-416 from 459353 to 1073578 tid 1 2014-07-01 16:52:24,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0437250-416 from 1871203 to 2373642 tid 2 2014-07-01 16:52:24,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: writing mira0437250-416 from 3029822 to 3259353 tid 3 2014-07-01 16:52:24,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:24,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: finishing write tid 4 to mira0437250-402 2014-07-01 16:52:24,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 59 (ObjNum 401 snap 0 seq_num 401) dirty exists 2014-07-01 16:52:24,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 3 to mira0437250-401 2014-07-01 16:52:24,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:402: done (15 left) 2014-07-01 16:52:24,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: write initial oid 417 2014-07-01 16:52:24,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: seq_num 416 ranges {555606=586030,1833911=421211,2776777=178829} 2014-07-01 16:52:24,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: writing mira0437250-417 from 555606 to 1141636 tid 1 2014-07-01 16:52:24,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2416: expect (ObjNum 157 snap 0 seq_num 157) 2014-07-01 16:52:24,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: writing mira0437250-417 from 1833911 to 2255122 tid 2 2014-07-01 16:52:24,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: writing mira0437250-417 from 2776777 to 2955606 tid 3 2014-07-01 16:52:24,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:24,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 5 to mira0437250-401 2014-07-01 16:52:24,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: finishing write tid 6 to mira0437250-401 2014-07-01 16:52:24,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 111 (ObjNum 400 snap 0 seq_num 400) dirty exists 2014-07-01 16:52:24,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:401: done (15 left) 2014-07-01 16:52:24,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: write initial oid 418 2014-07-01 16:52:24,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: seq_num 417 ranges {499412=442506,1396619=642830,2729426=613594,3699411=1} 2014-07-01 16:52:24,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: writing mira0437250-418 from 499412 to 941918 tid 1 2014-07-01 16:52:24,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: writing mira0437250-418 from 1396619 to 2039449 tid 2 2014-07-01 16:52:24,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: writing mira0437250-418 from 2729426 to 3343020 tid 3 2014-07-01 16:52:24,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: writing mira0437250-418 from 3699411 to 3699412 tid 4 2014-07-01 16:52:24,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:25,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 1 to mira0437250-403 2014-07-01 16:52:25,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 2 to mira0437250-403 2014-07-01 16:52:25,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 4 to mira0437250-403 2014-07-01 16:52:25,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 1 to mira0437250-404 2014-07-01 16:52:25,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 3 to mira0437250-404 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 1 to mira0437250-405 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 3 to mira0437250-405 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 1 to mira0437250-411 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 2 to mira0437250-411 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 3 to mira0437250-411 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 4 to mira0437250-411 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 6 to mira0437250-411 2014-07-01 16:52:25,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 1 to mira0437250-406 2014-07-01 16:52:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 1 to mira0437250-414 2014-07-01 16:52:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 2 to mira0437250-414 2014-07-01 16:52:25,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 4 to mira0437250-414 2014-07-01 16:52:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 2 to mira0437250-406 2014-07-01 16:52:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 3 to mira0437250-406 2014-07-01 16:52:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 5 to mira0437250-406 2014-07-01 16:52:25,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 1 to mira0437250-407 2014-07-01 16:52:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 2 to mira0437250-407 2014-07-01 16:52:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 3 to mira0437250-407 2014-07-01 16:52:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 5 to mira0437250-407 2014-07-01 16:52:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 1 to mira0437250-408 2014-07-01 16:52:25,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 2 to mira0437250-408 2014-07-01 16:52:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 4 to mira0437250-408 2014-07-01 16:52:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 1 to mira0437250-409 2014-07-01 16:52:25,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 3 to mira0437250-409 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: finishing write tid 5 to mira0437250-403 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 77 (ObjNum 402 snap 0 seq_num 402) dirty exists 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:403: done (15 left) 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: write initial oid 419 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: seq_num 418 ranges {443217=1} 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: writing mira0437250-419 from 443217 to 443218 tid 1 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 1 to mira0437250-412 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 2 to mira0437250-412 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 1 to mira0437250-410 2014-07-01 16:52:25,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 4 to mira0437250-412 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 2 to mira0437250-410 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 3 to mira0437250-410 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 1 to mira0437250-413 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 2 to mira0437250-413 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 4 to mira0437250-413 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 1 to mira0437250-415 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 2 to mira0437250-415 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 3 to mira0437250-415 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 4 to mira0437250-410 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 5 to mira0437250-415 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 1 to mira0437250-416 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 2 to mira0437250-416 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 3 to mira0437250-416 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 5 to mira0437250-416 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 1 to mira0437250-417 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 2 to mira0437250-417 2014-07-01 16:52:25,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 6 to mira0437250-410 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 3 to mira0437250-417 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 5 to mira0437250-417 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: finishing write tid 4 to mira0437250-404 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 79 (ObjNum 403 snap 0 seq_num 403) dirty exists 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: finishing write tid 7 to mira0437250-410 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 93 (ObjNum 409 snap 0 seq_num 409) dirty exists 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:404: done (15 left) 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:410: done (14 left) 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: write initial oid 420 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: seq_num 419 ranges {787024=400000} 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: writing mira0437250-420 from 787024 to 1187024 tid 1 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: write initial oid 421 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: seq_num 420 ranges {599629=662815,1399628=1} 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: writing mira0437250-421 from 599629 to 1262444 tid 1 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: writing mira0437250-421 from 1399628 to 1399629 tid 2 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 1 to mira0437250-418 2014-07-01 16:52:25,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: finishing write tid 4 to mira0437250-405 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 97 (ObjNum 404 snap 0 seq_num 404) dirty exists 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 2 to mira0437250-418 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 3 to mira0437250-418 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 4 to mira0437250-418 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 6 to mira0437250-418 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: finishing write tid 6 to mira0437250-406 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 406 v 77 (ObjNum 405 snap 0 seq_num 405) dirty exists 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: finishing write tid 7 to mira0437250-418 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 98 (ObjNum 417 snap 0 seq_num 417) dirty exists 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: finishing write tid 6 to mira0437250-407 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 407 v 60 (ObjNum 406 snap 0 seq_num 406) dirty exists 2014-07-01 16:52:25,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 1 to mira0437250-419 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: finishing write tid 4 to mira0437250-409 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 62 (ObjNum 408 snap 0 seq_num 408) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 3 to mira0437250-419 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: finishing write tid 5 to mira0437250-408 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 76 (ObjNum 407 snap 0 seq_num 407) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: finishing write tid 7 to mira0437250-411 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 102 (ObjNum 410 snap 0 seq_num 410) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: finishing write tid 5 to mira0437250-412 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 412 v 105 (ObjNum 411 snap 0 seq_num 411) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: finishing write tid 5 to mira0437250-414 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 105 (ObjNum 413 snap 0 seq_num 413) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: finishing write tid 5 to mira0437250-413 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 65 (ObjNum 412 snap 0 seq_num 412) dirty exists 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: finishing write tid 6 to mira0437250-415 2014-07-01 16:52:25,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 415 v 63 (ObjNum 414 snap 0 seq_num 414) dirty exists 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 1 to mira0437250-420 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 3 to mira0437250-420 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: finishing write tid 6 to mira0437250-416 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 69 (ObjNum 415 snap 0 seq_num 415) dirty exists 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: finishing write tid 4 to mira0437250-420 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 105 (ObjNum 419 snap 0 seq_num 419) dirty exists 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: finishing write tid 6 to mira0437250-417 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 67 (ObjNum 416 snap 0 seq_num 416) dirty exists 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: finishing write tid 4 to mira0437250-419 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 107 (ObjNum 418 snap 0 seq_num 418) dirty exists 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:405: done (15 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:406: done (14 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:407: done (13 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:408: done (12 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:409: done (11 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:411: done (10 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:412: done (9 left) 2014-07-01 16:52:25,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:413: done (8 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:414: done (7 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:415: done (6 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:416: done (5 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:417: done (4 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:418: done (3 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:419: done (2 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:420: done (1 left) 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: write initial oid 422 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: seq_num 421 ranges {544459=518267,1665627=426459,2144458=1} 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: writing mira0437250-422 from 544459 to 1062726 tid 1 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: writing mira0437250-422 from 1665627 to 2092086 tid 2 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: writing mira0437250-422 from 2144458 to 2144459 tid 3 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: write initial oid 423 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: seq_num 422 ranges {488265=641496,1743664=745230,2888264=1} 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0437250-423 from 488265 to 1129761 tid 1 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0437250-423 from 1743664 to 2488894 tid 2 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: writing mira0437250-423 from 2888264 to 2888265 tid 3 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: write initial oid 424 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: seq_num 423 ranges {584518=613300,1857795=566849,2584517=1} 2014-07-01 16:52:25,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: writing mira0437250-424 from 584518 to 1197818 tid 1 2014-07-01 16:52:25,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: writing mira0437250-424 from 1857795 to 2424644 tid 2 2014-07-01 16:52:25,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: writing mira0437250-424 from 2584517 to 2584518 tid 3 2014-07-01 16:52:25,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: write initial oid 425 2014-07-01 16:52:25,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: seq_num 424 ranges {644676=737552,2088279=520692,3168402=542536,3844675=1} 2014-07-01 16:52:25,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: finishing write tid 1 to mira0435601-109 2014-07-01 16:52:25,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: finishing write tid 2 to mira0435601-109 2014-07-01 16:52:25,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: finishing write tid 4 to mira0435601-109 2014-07-01 16:52:25,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 1 to mira0435601-456 2014-07-01 16:52:25,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 2 to mira0435601-456 2014-07-01 16:52:25,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 3 to mira0435601-456 2014-07-01 16:52:25,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 4 to mira0435601-456 2014-07-01 16:52:25,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 6 to mira0435601-456 2014-07-01 16:52:25,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: finishing write tid 7 to mira0435601-456 2014-07-01 16:52:25,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 217 (ObjNum 1558 snap 0 seq_num 1558) dirty exists 2014-07-01 16:52:25,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: finishing write tid 5 to mira0435601-109 2014-07-01 16:52:25,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 212 (ObjNum 1557 snap 0 seq_num 1557) dirty exists 2014-07-01 16:52:25,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: done (14 left) 2014-07-01 16:52:25,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: done (13 left) 2014-07-01 16:52:25,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: done (12 left) 2014-07-01 16:52:25,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: done (11 left) 2014-07-01 16:52:25,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: done (10 left) 2014-07-01 16:52:25,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: done (9 left) 2014-07-01 16:52:25,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: done (8 left) 2014-07-01 16:52:25,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: done (7 left) 2014-07-01 16:52:25,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2415: done (6 left) 2014-07-01 16:52:25,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2416: done (5 left) 2014-07-01 16:52:25,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2417: done (4 left) 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: done (3 left) 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: done (2 left) 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: done (1 left) 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: done (0 left) 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 479 current snap is 0 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: seq_num 1562 ranges {0=753664,753664=729088,1482752=73728} 2014-07-01 16:52:25,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: writing mira0435601-479 from 0 to 753664 tid 1 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: writing mira0435601-479 from 753664 to 1482752 tid 2 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: writing mira0435601-479 from 1482752 to 1556480 tid 3 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 12 current snap is 0 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: seq_num 1563 ranges {2658304=626688,3284992=712704,3997696=90112} 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: writing mira0435601-12 from 2658304 to 3284992 tid 1 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: writing mira0435601-12 from 3284992 to 3997696 tid 2 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: writing mira0435601-12 from 3997696 to 4087808 tid 3 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 88 current snap is 0 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: seq_num 1564 ranges {458752=249856} 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: writing mira0435601-88 from 458752 to 708608 tid 1 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 155 current snap is 0 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: finishing write tid 1 to mira0435601-479 2014-07-01 16:52:25,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: finishing write tid 2 to mira0435601-479 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: finishing write tid 3 to mira0435601-479 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: finishing write tid 5 to mira0435601-479 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: finishing write tid 6 to mira0435601-479 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 233 (ObjNum 1562 snap 0 seq_num 1562) dirty exists 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 1 to mira0435601-12 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: seq_num 1565 ranges {36864=573440,610304=401408} 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: writing mira0435601-155 from 36864 to 610304 tid 1 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0437250-425 from 644676 to 1382228 tid 1 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0437250-425 from 2088279 to 2608971 tid 2 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0437250-425 from 3168402 to 3710938 tid 3 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: writing mira0437250-425 from 3844675 to 3844676 tid 4 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: write initial oid 426 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: seq_num 425 ranges {589505=1} 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: writing mira0437250-426 from 589505 to 589506 tid 1 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: write initial oid 427 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: seq_num 426 ranges {284735=1} 2014-07-01 16:52:25,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: writing mira0437250-427 from 284735 to 284736 tid 1 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: write initial oid 428 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: seq_num 427 ranges {628541=400000} 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: writing mira0437250-428 from 628541 to 1028541 tid 1 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: write initial oid 429 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: seq_num 428 ranges {573371=660866,1653416=119955} 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: writing mira0437250-429 from 573371 to 1234237 tid 1 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: writing mira0437250-429 from 1653416 to 1773371 tid 2 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: write initial oid 430 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: seq_num 429 ranges {784953=516318,1984952=1} 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0437250-430 from 784953 to 1301271 tid 1 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: writing mira0437250-430 from 1984952 to 1984953 tid 2 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: write initial oid 431 2014-07-01 16:52:25,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: seq_num 430 ranges {481207=639546,1597008=84199} 2014-07-01 16:52:25,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: writing mira0437250-431 from 481207 to 1120753 tid 1 2014-07-01 16:52:25,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: writing mira0437250-431 from 1597008 to 1681207 tid 2 2014-07-01 16:52:25,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: write initial oid 432 2014-07-01 16:52:25,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: seq_num 431 ranges {674612=612375,2057893=666330,3269460=205152} 2014-07-01 16:52:25,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: writing mira0435601-155 from 610304 to 1011712 tid 2 2014-07-01 16:52:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: done (3 left) 2014-07-01 16:52:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: read oid 220 snap -1 2014-07-01 16:52:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 2 to mira0435601-12 2014-07-01 16:52:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 3 to mira0435601-12 2014-07-01 16:52:25,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 5 to mira0435601-12 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 6 to mira0435601-12 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 106 (ObjNum 1563 snap 0 seq_num 1563) dirty exists 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: finishing write tid 1 to mira0435601-88 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: finishing write tid 3 to mira0435601-88 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: finishing write tid 4 to mira0435601-88 2014-07-01 16:52:25,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 214 (ObjNum 1564 snap 0 seq_num 1564) dirty exists 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: done (3 left) 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: done (2 left) 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2427: read oid 196 snap -1 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2428: read oid 482 snap -1 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: read oid 263 snap -1 2014-07-01 16:52:25,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 232 current snap is 0 2014-07-01 16:52:25,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: seq_num 1566 ranges {1953792=602112,2555904=708608,3264512=471040,3735552=425984} 2014-07-01 16:52:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0437250-432 from 674612 to 1286987 tid 1 2014-07-01 16:52:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0437250-432 from 2057893 to 2724223 tid 2 2014-07-01 16:52:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: writing mira0437250-432 from 3269460 to 3474612 tid 3 2014-07-01 16:52:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: write initial oid 433 2014-07-01 16:52:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: seq_num 432 ranges {769842=735603,2038777=486926,3087838=82004} 2014-07-01 16:52:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: writing mira0437250-433 from 769842 to 1505445 tid 1 2014-07-01 16:52:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: writing mira0437250-433 from 2038777 to 2525703 tid 2 2014-07-01 16:52:25,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: writing mira0435601-232 from 1953792 to 2555904 tid 1 2014-07-01 16:52:25,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: writing mira0435601-232 from 2555904 to 3264512 tid 2 2014-07-01 16:52:25,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: writing mira0435601-232 from 3264512 to 3735552 tid 3 2014-07-01 16:52:25,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: writing mira0435601-232 from 3735552 to 4161536 tid 4 2014-07-01 16:52:25,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: read oid 495 snap -1 2014-07-01 16:52:25,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: finishing write tid 1 to mira0435601-155 2014-07-01 16:52:25,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: expect (ObjNum 1457 snap 0 seq_num 1457) 2014-07-01 16:52:25,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: writing mira0437250-433 from 3087838 to 3169842 tid 3 2014-07-01 16:52:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: write initial oid 434 2014-07-01 16:52:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: seq_num 433 ranges {582448=591055,1752909=689345,3056359=326089} 2014-07-01 16:52:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: writing mira0437250-434 from 582448 to 1173503 tid 1 2014-07-01 16:52:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: writing mira0437250-434 from 1752909 to 2442254 tid 2 2014-07-01 16:52:25,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: writing mira0437250-434 from 3056359 to 3382448 tid 3 2014-07-01 16:52:25,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: write initial oid 435 2014-07-01 16:52:25,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: seq_num 434 ranges {773806=400000} 2014-07-01 16:52:25,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: writing mira0437250-435 from 773806 to 1173806 tid 1 2014-07-01 16:52:25,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: write initial oid 436 2014-07-01 16:52:25,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: seq_num 435 ranges {470059=400000} 2014-07-01 16:52:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: writing mira0437250-436 from 470059 to 870059 tid 1 2014-07-01 16:52:25,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:25,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 1 to mira0437250-421 2014-07-01 16:52:25,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 2 to mira0437250-421 2014-07-01 16:52:25,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 4 to mira0437250-421 2014-07-01 16:52:25,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: finishing write tid 5 to mira0437250-421 2014-07-01 16:52:25,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 110 (ObjNum 420 snap 0 seq_num 420) dirty exists 2014-07-01 16:52:25,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 1 to mira0437250-422 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 2 to mira0437250-422 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 3 to mira0437250-422 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 5 to mira0437250-422 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: finishing write tid 6 to mira0437250-422 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 80 (ObjNum 421 snap 0 seq_num 421) dirty exists 2014-07-01 16:52:25,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 1 to mira0437250-423 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 2 to mira0437250-423 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 3 to mira0437250-423 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 5 to mira0437250-423 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: finishing write tid 6 to mira0437250-423 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 102 (ObjNum 422 snap 0 seq_num 422) dirty exists 2014-07-01 16:52:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 1 to mira0437250-424 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 2 to mira0437250-424 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 3 to mira0437250-424 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 5 to mira0437250-424 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:421: done (15 left) 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:422: done (14 left) 2014-07-01 16:52:25,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:423: done (13 left) 2014-07-01 16:52:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: write initial oid 437 2014-07-01 16:52:25,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: seq_num 436 ranges {566312=1} 2014-07-01 16:52:25,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: writing mira0437250-437 from 566312 to 566313 tid 1 2014-07-01 16:52:25,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: write initial oid 438 2014-07-01 16:52:25,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: seq_num 437 ranges {758695=514369,1965598=393097} 2014-07-01 16:52:25,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: writing mira0437250-438 from 758695 to 1273064 tid 1 2014-07-01 16:52:25,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: writing mira0437250-438 from 1965598 to 2358695 tid 2 2014-07-01 16:52:25,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: write initial oid 439 2014-07-01 16:52:25,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: seq_num 438 ranges {571301=638621,1948530=543169,2571300=1} 2014-07-01 16:52:25,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0437250-439 from 571301 to 1209922 tid 1 2014-07-01 16:52:25,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0437250-439 from 1948530 to 2491699 tid 2 2014-07-01 16:52:25,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: writing mira0437250-439 from 2571300 to 2571301 tid 3 2014-07-01 16:52:25,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:25,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: finishing write tid 6 to mira0437250-424 2014-07-01 16:52:25,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 106 (ObjNum 423 snap 0 seq_num 423) dirty exists 2014-07-01 16:52:25,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 1 to mira0437250-425 2014-07-01 16:52:25,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 2 to mira0437250-425 2014-07-01 16:52:25,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 1 to mira0437250-427 2014-07-01 16:52:25,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 3 to mira0437250-427 2014-07-01 16:52:25,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 1 to mira0437250-426 2014-07-01 16:52:25,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 3 to mira0437250-425 2014-07-01 16:52:25,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 3 to mira0437250-426 2014-07-01 16:52:25,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 4 to mira0437250-425 2014-07-01 16:52:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 6 to mira0437250-425 2014-07-01 16:52:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: finishing write tid 4 to mira0437250-426 2014-07-01 16:52:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 426 v 108 (ObjNum 425 snap 0 seq_num 425) dirty exists 2014-07-01 16:52:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: finishing write tid 4 to mira0437250-427 2014-07-01 16:52:25,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 107 (ObjNum 426 snap 0 seq_num 426) dirty exists 2014-07-01 16:52:25,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: finishing write tid 7 to mira0437250-425 2014-07-01 16:52:25,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 74 (ObjNum 424 snap 0 seq_num 424) dirty exists 2014-07-01 16:52:25,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 1 to mira0437250-428 2014-07-01 16:52:25,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 3 to mira0437250-428 2014-07-01 16:52:25,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: finishing write tid 4 to mira0437250-428 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 67 (ObjNum 427 snap 0 seq_num 427) dirty exists 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 1 to mira0437250-429 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 2 to mira0437250-429 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 4 to mira0437250-429 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: finishing write tid 5 to mira0437250-429 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 82 (ObjNum 428 snap 0 seq_num 428) dirty exists 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 1 to mira0437250-430 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 2 to mira0437250-430 2014-07-01 16:52:25,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:424: done (15 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:425: done (14 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:426: done (13 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:427: done (12 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:428: done (11 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:429: done (10 left) 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: write initial oid 440 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: seq_num 439 ranges {666530=610425,2061637=204893} 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: writing mira0437250-440 from 666530 to 1276955 tid 1 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: writing mira0437250-440 from 2061637 to 2266530 tid 2 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: write initial oid 441 2014-07-01 16:52:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: seq_num 440 ranges {611360=733654,2140698=434983,3011359=1} 2014-07-01 16:52:25,993.993 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:52:25,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: writing mira0437250-441 from 611360 to 1345014 tid 1 2014-07-01 16:52:25,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: writing mira0437250-441 from 2140698 to 2575681 tid 2 2014-07-01 16:52:25,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: writing mira0437250-441 from 3011359 to 3011360 tid 3 2014-07-01 16:52:25,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: write initial oid 442 2014-07-01 16:52:25,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: seq_num 441 ranges {556190=590130,1588078=505178,2710066=574071,3756189=1} 2014-07-01 16:52:26,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: writing mira0437250-442 from 556190 to 1146320 tid 1 2014-07-01 16:52:26,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: writing mira0437250-442 from 1588078 to 2093256 tid 2 2014-07-01 16:52:26,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: writing mira0437250-442 from 2710066 to 3284137 tid 3 2014-07-01 16:52:26,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: writing mira0437250-442 from 3756189 to 3756190 tid 4 2014-07-01 16:52:26,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: write initial oid 443 2014-07-01 16:52:26,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: seq_num 442 ranges {766747=713359,2084290=610445,3460667=506080} 2014-07-01 16:52:26,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: writing mira0437250-443 from 766747 to 1480106 tid 1 2014-07-01 16:52:26,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: writing mira0437250-443 from 2084290 to 2694735 tid 2 2014-07-01 16:52:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: writing mira0437250-443 from 3460667 to 3966747 tid 3 2014-07-01 16:52:26,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: write initial oid 444 2014-07-01 16:52:26,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: seq_num 443 ranges {711576=1} 2014-07-01 16:52:26,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: writing mira0437250-444 from 711576 to 711577 tid 1 2014-07-01 16:52:26,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: write initial oid 445 2014-07-01 16:52:26,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: seq_num 444 ranges {655383=408391,1455382=1} 2014-07-01 16:52:26,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: writing mira0437250-445 from 655383 to 1063774 tid 1 2014-07-01 16:52:26,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: writing mira0437250-445 from 1455382 to 1455383 tid 2 2014-07-01 16:52:26,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 4 to mira0437250-430 2014-07-01 16:52:26,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: finishing write tid 5 to mira0437250-430 2014-07-01 16:52:26,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 80 (ObjNum 429 snap 0 seq_num 429) dirty exists 2014-07-01 16:52:26,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 1 to mira0437250-431 2014-07-01 16:52:26,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 2 to mira0437250-431 2014-07-01 16:52:26,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 4 to mira0437250-431 2014-07-01 16:52:26,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:430: done (15 left) 2014-07-01 16:52:26,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: write initial oid 446 2014-07-01 16:52:26,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: seq_num 445 ranges {751637=400000} 2014-07-01 16:52:26,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: writing mira0437250-446 from 751637 to 1151637 tid 1 2014-07-01 16:52:26,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: finishing write tid 5 to mira0437250-431 2014-07-01 16:52:26,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 82 (ObjNum 430 snap 0 seq_num 430) dirty exists 2014-07-01 16:52:26,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 1 to mira0437250-432 2014-07-01 16:52:26,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 2 to mira0437250-432 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 3 to mira0437250-432 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 5 to mira0437250-432 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: finishing write tid 6 to mira0437250-432 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 92 (ObjNum 431 snap 0 seq_num 431) dirty exists 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 1 to mira0437250-437 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 3 to mira0437250-437 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 1 to mira0437250-436 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: finishing write tid 4 to mira0437250-437 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 83 (ObjNum 436 snap 0 seq_num 436) dirty exists 2014-07-01 16:52:26,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 3 to mira0437250-436 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: finishing write tid 4 to mira0437250-436 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 82 (ObjNum 435 snap 0 seq_num 435) dirty exists 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 1 to mira0437250-438 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 2 to mira0437250-438 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 4 to mira0437250-438 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: finishing write tid 5 to mira0437250-438 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 95 (ObjNum 437 snap 0 seq_num 437) dirty exists 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 1 to mira0437250-439 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 2 to mira0437250-439 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:431: done (15 left) 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:432: done (14 left) 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:436: done (13 left) 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:437: done (12 left) 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:438: done (11 left) 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: write initial oid 447 2014-07-01 16:52:26,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: seq_num 446 ranges {696466=788096,1896465=1} 2014-07-01 16:52:26,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: writing mira0437250-447 from 696466 to 1484562 tid 1 2014-07-01 16:52:26,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: writing mira0437250-447 from 1896465 to 1896466 tid 2 2014-07-01 16:52:26,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: write initial oid 448 2014-07-01 16:52:26,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: seq_num 447 ranges {756624=512348,2032358=712845,3156623=1} 2014-07-01 16:52:26,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0437250-448 from 756624 to 1268972 tid 1 2014-07-01 16:52:26,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0437250-448 from 2032358 to 2745203 tid 2 2014-07-01 16:52:26,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: writing mira0437250-448 from 3156623 to 3156624 tid 3 2014-07-01 16:52:26,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: write initial oid 449 2014-07-01 16:52:26,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: seq_num 448 ranges {452878=733753,1596092=534465,2616821=236057} 2014-07-01 16:52:26,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: writing mira0437250-449 from 452878 to 1186631 tid 1 2014-07-01 16:52:26,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: writing mira0437250-449 from 1596092 to 2130557 tid 2 2014-07-01 16:52:26,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: writing mira0437250-449 from 2616821 to 2852878 tid 3 2014-07-01 16:52:26,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: write initial oid 450 2014-07-01 16:52:26,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: seq_num 449 ranges {796684=740629,1992848=604660,3100670=496014} 2014-07-01 16:52:26,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: writing mira0437250-450 from 796684 to 1537313 tid 1 2014-07-01 16:52:26,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: writing mira0437250-450 from 1992848 to 2597508 tid 2 2014-07-01 16:52:26,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: writing mira0437250-450 from 3100670 to 3596684 tid 3 2014-07-01 16:52:26,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: write initial oid 451 2014-07-01 16:52:26,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: seq_num 450 ranges {340488=1} 2014-07-01 16:52:26,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: writing mira0437250-451 from 340488 to 340489 tid 1 2014-07-01 16:52:26,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 3 to mira0437250-439 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 5 to mira0437250-439 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: finishing write tid 6 to mira0437250-439 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 78 (ObjNum 438 snap 0 seq_num 438) dirty exists 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 1 to mira0437250-440 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:439: done (15 left) 2014-07-01 16:52:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: write initial oid 452 2014-07-01 16:52:26,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: seq_num 451 ranges {553094=1} 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: writing mira0437250-452 from 553094 to 553095 tid 1 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 2 to mira0437250-440 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 4 to mira0437250-440 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: finishing write tid 5 to mira0437250-440 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 85 (ObjNum 439 snap 0 seq_num 439) dirty exists 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 1 to mira0437250-441 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 2 to mira0437250-441 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 3 to mira0437250-441 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 5 to mira0437250-441 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: finishing write tid 6 to mira0437250-441 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 71 (ObjNum 440 snap 0 seq_num 440) dirty exists 2014-07-01 16:52:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:440: done (15 left) 2014-07-01 16:52:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:441: done (14 left) 2014-07-01 16:52:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: write initial oid 453 2014-07-01 16:52:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: seq_num 452 ranges {496901=408490,1296900=1} 2014-07-01 16:52:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: writing mira0437250-453 from 496901 to 905391 tid 1 2014-07-01 16:52:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: writing mira0437250-453 from 1296900 to 1296901 tid 2 2014-07-01 16:52:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: write initial oid 454 2014-07-01 16:52:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: seq_num 453 ranges {593155=400000} 2014-07-01 16:52:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: writing mira0437250-454 from 593155 to 993155 tid 1 2014-07-01 16:52:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 1 to mira0437250-443 2014-07-01 16:52:26,740.740 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:51:44.722257', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882477972, u'health': u'HEALTH_OK', u'kb_used': 14303080, u'store_stats': {u'bytes_total': 8804567, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561671}}, {u'last_updated': u'2014-07-01 07:51:54.539712', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886455256, u'health': u'HEALTH_OK', u'kb_used': 10325796, u'store_stats': {u'bytes_total': 12998348, u'bytes_log': 9371648, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561148}}, {u'last_updated': u'2014-07-01 07:52:11.322695', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886437844, u'health': u'HEALTH_OK', u'kb_used': 10343208, u'store_stats': {u'bytes_total': 15095560, u'bytes_log': 11468800, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3561208}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool rbd pg_num 64 > pgp_num 44'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 46 > pgp_num 26'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'num_pgs': 414, u'data_bytes': 2020366190, u'bytes_used': 4085096448, u'op_per_sec': 61, u'version': 1722, u'pgs_by_state': [{u'count': 414, u'state_name': u'active+clean'}], u'read_bytes_sec': 0, u'write_bytes_sec': 8231009, u'bytes_avail': 4994497441792}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 636, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:52:26,741.741 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:52:26,741.741 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:26,742.742 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:52:26,742.742 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:52:26,742.742 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:52:26,742.742 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:52:26,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 2 to mira0437250-443 2014-07-01 16:52:26,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 3 to mira0437250-443 2014-07-01 16:52:26,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 5 to mira0437250-443 2014-07-01 16:52:26,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 1 to mira0437250-446 2014-07-01 16:52:26,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 1 to mira0437250-451 2014-07-01 16:52:26,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 3 to mira0437250-446 2014-07-01 16:52:26,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 3 to mira0437250-451 2014-07-01 16:52:26,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: finishing write tid 4 to mira0437250-446 2014-07-01 16:52:26,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 85 (ObjNum 445 snap 0 seq_num 445) dirty exists 2014-07-01 16:52:26,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:446: done (15 left) 2014-07-01 16:52:26,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: write initial oid 455 2014-07-01 16:52:26,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: seq_num 454 ranges {536960=787171,1736959=1} 2014-07-01 16:52:26,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: expect (ObjNum 934 snap 0 seq_num 934) 2014-07-01 16:52:26,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: writing mira0437250-455 from 536960 to 1324131 tid 1 2014-07-01 16:52:26,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: writing mira0437250-455 from 1736959 to 1736960 tid 2 2014-07-01 16:52:26,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:26,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: finishing write tid 4 to mira0437250-451 2014-07-01 16:52:26,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 110 (ObjNum 450 snap 0 seq_num 450) dirty exists 2014-07-01 16:52:26,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:451: done (15 left) 2014-07-01 16:52:26,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: write initial oid 456 2014-07-01 16:52:26,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: seq_num 455 ranges {481790=510399,1769352=411303,2481789=1} 2014-07-01 16:52:26,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: writing mira0437250-456 from 481790 to 992189 tid 1 2014-07-01 16:52:26,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: writing mira0437250-456 from 1769352 to 2180655 tid 2 2014-07-01 16:52:26,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: writing mira0437250-456 from 2481789 to 2481790 tid 3 2014-07-01 16:52:26,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: finishing write tid 6 to mira0437250-443 2014-07-01 16:52:27,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 114 (ObjNum 442 snap 0 seq_num 442) dirty exists 2014-07-01 16:52:27,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:443: done (15 left) 2014-07-01 16:52:27,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: write initial oid 457 2014-07-01 16:52:27,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: seq_num 456 ranges {694396=483227,1849436=632922,2694395=1} 2014-07-01 16:52:27,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: writing mira0437250-457 from 694396 to 1177623 tid 1 2014-07-01 16:52:27,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: writing mira0437250-457 from 1849436 to 2482358 tid 2 2014-07-01 16:52:27,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: writing mira0437250-457 from 2694395 to 2694396 tid 3 2014-07-01 16:52:27,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 1 to mira0437250-455 2014-07-01 16:52:27,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 2 to mira0437250-455 2014-07-01 16:52:27,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 4 to mira0437250-455 2014-07-01 16:52:27,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 1 to mira0437250-447 2014-07-01 16:52:27,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 2 to mira0437250-447 2014-07-01 16:52:27,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 4 to mira0437250-447 2014-07-01 16:52:27,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 1 to mira0437250-457 2014-07-01 16:52:27,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 2 to mira0437250-457 2014-07-01 16:52:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 3 to mira0437250-457 2014-07-01 16:52:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 5 to mira0437250-457 2014-07-01 16:52:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 1 to mira0437250-453 2014-07-01 16:52:27,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 2 to mira0437250-453 2014-07-01 16:52:27,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 4 to mira0437250-453 2014-07-01 16:52:27,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: finishing write tid 5 to mira0437250-453 2014-07-01 16:52:27,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 83 (ObjNum 452 snap 0 seq_num 452) dirty exists 2014-07-01 16:52:27,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:453: done (15 left) 2014-07-01 16:52:27,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: write initial oid 458 2014-07-01 16:52:27,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: seq_num 457 ranges {638202=490104,1562545=551693,2903753=469215,3438201=1} 2014-07-01 16:52:27,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: writing mira0437250-458 from 638202 to 1128306 tid 1 2014-07-01 16:52:27,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: writing mira0437250-458 from 1562545 to 2114238 tid 2 2014-07-01 16:52:27,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: writing mira0437250-458 from 2903753 to 3372968 tid 3 2014-07-01 16:52:27,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: writing mira0437250-458 from 3438201 to 3438202 tid 4 2014-07-01 16:52:27,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: finishing write tid 5 to mira0437250-455 2014-07-01 16:52:27,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 117 (ObjNum 454 snap 0 seq_num 454) dirty exists 2014-07-01 16:52:27,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: finishing write tid 6 to mira0437250-457 2014-07-01 16:52:27,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 75 (ObjNum 456 snap 0 seq_num 456) dirty exists 2014-07-01 16:52:27,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 1 to mira0437250-458 2014-07-01 16:52:27,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 2 to mira0437250-458 2014-07-01 16:52:27,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:455: done (15 left) 2014-07-01 16:52:27,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:457: done (14 left) 2014-07-01 16:52:27,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: write initial oid 459 2014-07-01 16:52:27,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: seq_num 458 ranges {183030=1} 2014-07-01 16:52:27,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: writing mira0437250-459 from 183030 to 183031 tid 1 2014-07-01 16:52:27,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: write initial oid 460 2014-07-01 16:52:27,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: seq_num 459 ranges {677237=434736,1734488=594932,2753755=737900,3877236=1} 2014-07-01 16:52:27,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0437250-460 from 677237 to 1111973 tid 1 2014-07-01 16:52:27,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0437250-460 from 1734488 to 2329420 tid 2 2014-07-01 16:52:27,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0437250-460 from 2753755 to 3491655 tid 3 2014-07-01 16:52:27,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: writing mira0437250-460 from 3877236 to 3877237 tid 4 2014-07-01 16:52:27,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 3 to mira0437250-458 2014-07-01 16:52:27,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 4 to mira0437250-458 2014-07-01 16:52:27,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 6 to mira0437250-458 2014-07-01 16:52:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: finishing write tid 7 to mira0437250-458 2014-07-01 16:52:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 115 (ObjNum 457 snap 0 seq_num 457) dirty exists 2014-07-01 16:52:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 1 to mira0437250-460 2014-07-01 16:52:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 2 to mira0437250-460 2014-07-01 16:52:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:458: done (15 left) 2014-07-01 16:52:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: write initial oid 461 2014-07-01 16:52:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: seq_num 460 ranges {738419=400000} 2014-07-01 16:52:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: writing mira0437250-461 from 738419 to 1138419 tid 1 2014-07-01 16:52:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 3 to mira0437250-460 2014-07-01 16:52:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 4 to mira0437250-460 2014-07-01 16:52:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 6 to mira0437250-460 2014-07-01 16:52:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: finishing write tid 7 to mira0437250-460 2014-07-01 16:52:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 90 (ObjNum 459 snap 0 seq_num 459) dirty exists 2014-07-01 16:52:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:460: done (15 left) 2014-07-01 16:52:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: write initial oid 462 2014-07-01 16:52:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: seq_num 461 ranges {683248=529769,1644032=239216} 2014-07-01 16:52:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0437250-462 from 683248 to 1213017 tid 1 2014-07-01 16:52:27,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: writing mira0437250-462 from 1644032 to 1883248 tid 2 2014-07-01 16:52:27,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 1 to mira0437250-461 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 3 to mira0437250-461 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: finishing write tid 4 to mira0437250-461 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 87 (ObjNum 460 snap 0 seq_num 460) dirty exists 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:461: done (15 left) 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: write initial oid 463 2014-07-01 16:52:27,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: seq_num 462 ranges {778478=785221,1578477=1} 2014-07-01 16:52:27,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: writing mira0437250-463 from 778478 to 1563699 tid 1 2014-07-01 16:52:27,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: writing mira0437250-463 from 1578477 to 1578478 tid 2 2014-07-01 16:52:27,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:27,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: done (6 left) 2014-07-01 16:52:27,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2427: done (5 left) 2014-07-01 16:52:27,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: done (4 left) 2014-07-01 16:52:27,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 347 current snap is 0 2014-07-01 16:52:27,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: seq_num 1567 ranges {2973696=471040,3444736=692224,4136960=745472,4882432=167936} 2014-07-01 16:52:28,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: writing mira0435601-347 from 2973696 to 3444736 tid 1 2014-07-01 16:52:28,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: writing mira0435601-347 from 3444736 to 4136960 tid 2 2014-07-01 16:52:28,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: writing mira0435601-347 from 4136960 to 4882432 tid 3 2014-07-01 16:52:28,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: writing mira0435601-347 from 4882432 to 5050368 tid 4 2014-07-01 16:52:28,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 56 current snap is 0 2014-07-01 16:52:28,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: seq_num 1568 ranges {0=745472,745472=475136,1220608=573440,1794048=155648} 2014-07-01 16:52:28,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: writing mira0435601-56 from 0 to 745472 tid 1 2014-07-01 16:52:28,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: writing mira0435601-56 from 745472 to 1220608 tid 2 2014-07-01 16:52:28,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: writing mira0435601-56 from 1220608 to 1794048 tid 3 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: writing mira0435601-56 from 1794048 to 1949696 tid 4 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: read oid 105 snap -1 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 1 to mira0435601-347 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: seq_num 1569 ranges {3338240=421888,3760128=458752,4218880=536576,4755456=208896} 2014-07-01 16:52:28,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: writing mira0435601-14 from 3338240 to 3760128 tid 1 2014-07-01 16:52:28,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: writing mira0435601-14 from 3760128 to 4218880 tid 2 2014-07-01 16:52:28,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: writing mira0435601-14 from 4218880 to 4755456 tid 3 2014-07-01 16:52:28,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: writing mira0435601-14 from 4755456 to 4964352 tid 4 2014-07-01 16:52:28,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2436: read oid 330 snap -1 2014-07-01 16:52:28,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 2 to mira0435601-347 2014-07-01 16:52:28,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2437: read oid 416 snap -1 2014-07-01 16:52:28,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 3 to mira0435601-347 2014-07-01 16:52:28,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 4 to mira0435601-347 2014-07-01 16:52:28,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 6 to mira0435601-347 2014-07-01 16:52:28,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: read oid 364 snap -1 2014-07-01 16:52:28,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 16:52:28,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: seq_num 1570 ranges {4608000=692224,5300224=688128,5988352=114688} 2014-07-01 16:52:28,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: writing mira0435601-97 from 4608000 to 5300224 tid 1 2014-07-01 16:52:28,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: writing mira0435601-97 from 5300224 to 5988352 tid 2 2014-07-01 16:52:28,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: writing mira0435601-97 from 5988352 to 6103040 tid 3 2014-07-01 16:52:28,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 151 current snap is 0 2014-07-01 16:52:28,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: seq_num 1571 ranges {1875968=565248,2441216=675840,3117056=126976} 2014-07-01 16:52:28,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0435601-151 from 1875968 to 2441216 tid 1 2014-07-01 16:52:28,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0435601-151 from 2441216 to 3117056 tid 2 2014-07-01 16:52:28,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0435601-151 from 3117056 to 3244032 tid 3 2014-07-01 16:52:28,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2441: read oid 285 snap -1 2014-07-01 16:52:28,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2436: expect (ObjNum 614 snap 0 seq_num 614) 2014-07-01 16:52:29,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:2436: done (13 left) 2014-07-01 16:52:29,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:52:29,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: seq_num 1572 ranges {4501504=192512} 2014-07-01 16:52:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: writing mira0435601-200 from 4501504 to 4694016 tid 1 2014-07-01 16:52:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 127 current snap is 0 2014-07-01 16:52:29,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: seq_num 1573 ranges {3883008=61440} 2014-07-01 16:52:29,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 1 to mira0437250-444 2014-07-01 16:52:29,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 3 to mira0437250-444 2014-07-01 16:52:29,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 1 to mira0437250-445 2014-07-01 16:52:29,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 2 to mira0437250-445 2014-07-01 16:52:29,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 1 to mira0437250-433 2014-07-01 16:52:29,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: writing mira0435601-127 from 3883008 to 3944448 tid 1 2014-07-01 16:52:29,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 448 current snap is 0 2014-07-01 16:52:29,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: seq_num 1574 ranges {0=786432} 2014-07-01 16:52:29,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: writing mira0435601-448 from 0 to 786432 tid 1 2014-07-01 16:52:29,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 1 to mira0435601-200 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 3 to mira0435601-200 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: finishing write tid 2 to mira0435601-155 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:2428: expect (ObjNum 1100 snap 0 seq_num 1100) 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 2 to mira0437250-433 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 3 to mira0437250-433 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 5 to mira0437250-433 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 1 to mira0437250-449 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: finishing write tid 6 to mira0437250-433 2014-07-01 16:52:29,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 78 (ObjNum 432 snap 0 seq_num 432) dirty exists 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:433: done (15 left) 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: write initial oid 464 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: seq_num 463 ranges {571884=509474,1569449=510784,2590080=779142,3371883=1} 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0437250-464 from 571884 to 1081358 tid 1 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0437250-464 from 1569449 to 2080233 tid 2 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0437250-464 from 2590080 to 3369222 tid 3 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: writing mira0437250-464 from 3371883 to 3371884 tid 4 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 1 to mira0437250-435 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 3 to mira0437250-435 2014-07-01 16:52:29,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: finishing write tid 4 to mira0437250-435 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 80 (ObjNum 434 snap 0 seq_num 434) dirty exists 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 2 to mira0437250-449 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 4 to mira0437250-445 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 3 to mira0437250-449 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 5 to mira0437250-449 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 1 to mira0437250-459 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 3 to mira0437250-459 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 1 to mira0437250-434 2014-07-01 16:52:29,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 2 to mira0437250-434 2014-07-01 16:52:29,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 3 to mira0437250-434 2014-07-01 16:52:29,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 5 to mira0437250-434 2014-07-01 16:52:29,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 1 to mira0437250-450 2014-07-01 16:52:29,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 2 to mira0437250-450 2014-07-01 16:52:29,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 3 to mira0437250-450 2014-07-01 16:52:29,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 5 to mira0437250-450 2014-07-01 16:52:29,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: finishing write tid 6 to mira0437250-450 2014-07-01 16:52:29,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 450 v 84 (ObjNum 449 snap 0 seq_num 449) dirty exists 2014-07-01 16:52:29,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 1 to mira0437250-442 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 2 to mira0437250-442 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 3 to mira0437250-442 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:435: done (15 left) 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:450: done (14 left) 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: write initial oid 465 2014-07-01 16:52:29,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: seq_num 464 ranges {667114=481278,1682557=580980,2790282=276832} 2014-07-01 16:52:29,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: writing mira0437250-465 from 667114 to 1148392 tid 1 2014-07-01 16:52:29,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: writing mira0437250-465 from 1682557 to 2263537 tid 2 2014-07-01 16:52:29,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: writing mira0437250-465 from 2790282 to 3067114 tid 3 2014-07-01 16:52:29,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: write initial oid 466 2014-07-01 16:52:29,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: seq_num 465 ranges {479719=604506,1629393=651175,3072787=206932} 2014-07-01 16:52:29,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: writing mira0437250-466 from 479719 to 1084225 tid 1 2014-07-01 16:52:29,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: writing mira0437250-466 from 1629393 to 2280568 tid 2 2014-07-01 16:52:29,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: writing mira0437250-466 from 3072787 to 3279719 tid 3 2014-07-01 16:52:29,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 4 to mira0437250-442 2014-07-01 16:52:29,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 6 to mira0437250-442 2014-07-01 16:52:29,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 1 to mira0437250-456 2014-07-01 16:52:29,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 2 to mira0437250-456 2014-07-01 16:52:29,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 3 to mira0437250-456 2014-07-01 16:52:29,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 5 to mira0437250-456 2014-07-01 16:52:29,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: finishing write tid 6 to mira0437250-456 2014-07-01 16:52:29,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 88 (ObjNum 455 snap 0 seq_num 455) dirty exists 2014-07-01 16:52:29,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 1 to mira0437250-448 2014-07-01 16:52:29,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 2 to mira0437250-448 2014-07-01 16:52:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 3 to mira0437250-448 2014-07-01 16:52:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 5 to mira0437250-448 2014-07-01 16:52:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 1 to mira0437250-452 2014-07-01 16:52:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 3 to mira0437250-452 2014-07-01 16:52:29,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 1 to mira0437250-454 2014-07-01 16:52:29,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 3 to mira0437250-454 2014-07-01 16:52:29,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: finishing write tid 6 to mira0437250-434 2014-07-01 16:52:29,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 80 (ObjNum 433 snap 0 seq_num 433) dirty exists 2014-07-01 16:52:29,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 1 to mira0437250-462 2014-07-01 16:52:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: finishing write tid 7 to mira0437250-442 2014-07-01 16:52:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 85 (ObjNum 441 snap 0 seq_num 441) dirty exists 2014-07-01 16:52:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 1 to mira0437250-464 2014-07-01 16:52:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 2 to mira0437250-464 2014-07-01 16:52:29,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:434: done (15 left) 2014-07-01 16:52:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:442: done (14 left) 2014-07-01 16:52:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:456: done (13 left) 2014-07-01 16:52:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: write initial oid 467 2014-07-01 16:52:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: seq_num 466 ranges {24548=1} 2014-07-01 16:52:29,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: writing mira0437250-467 from 24548 to 24549 tid 1 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: write initial oid 468 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: seq_num 467 ranges {767330=1} 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: writing mira0437250-468 from 767330 to 767331 tid 1 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: write initial oid 469 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: seq_num 468 ranges {463584=1} 2014-07-01 16:52:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: writing mira0437250-469 from 463584 to 463585 tid 1 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 3 to mira0437250-464 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 4 to mira0437250-464 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 6 to mira0437250-464 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: finishing write tid 6 to mira0437250-448 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 115 (ObjNum 447 snap 0 seq_num 447) dirty exists 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: finishing write tid 4 to mira0437250-452 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 117 (ObjNum 451 snap 0 seq_num 451) dirty exists 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:448: done (15 left) 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:452: done (14 left) 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: write initial oid 470 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: seq_num 469 ranges {523742=679244,1723741=1} 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0437250-470 from 523742 to 1202986 tid 1 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: writing mira0437250-470 from 1723741 to 1723742 tid 2 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: write initial oid 471 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: seq_num 470 ranges {468572=784296,1708662=759910} 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0437250-471 from 468572 to 1252868 tid 1 2014-07-01 16:52:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: writing mira0437250-471 from 1708662 to 2468572 tid 2 2014-07-01 16:52:29,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: finishing write tid 4 to mira0437250-454 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 69 (ObjNum 453 snap 0 seq_num 453) dirty exists 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: finishing write tid 7 to mira0437250-464 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 74 (ObjNum 463 snap 0 seq_num 463) dirty exists 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 1 to mira0437250-466 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 2 to mira0437250-466 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:454: done (15 left) 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:464: done (14 left) 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: write initial oid 472 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: seq_num 471 ranges {564826=507524,1822794=342032} 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: writing mira0437250-472 from 564826 to 1072350 tid 1 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: writing mira0437250-472 from 1822794 to 2164826 tid 2 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: write initial oid 473 2014-07-01 16:52:29,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: seq_num 472 ranges {508632=479329,1500831=679437,2593366=315266} 2014-07-01 16:52:29,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: writing mira0437250-473 from 508632 to 987961 tid 1 2014-07-01 16:52:29,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: writing mira0437250-473 from 1500831 to 2180268 tid 2 2014-07-01 16:52:29,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: writing mira0437250-473 from 2593366 to 2908632 tid 3 2014-07-01 16:52:29,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 3 to mira0437250-466 2014-07-01 16:52:29,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 5 to mira0437250-466 2014-07-01 16:52:29,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 1 to mira0437250-468 2014-07-01 16:52:29,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 3 to mira0437250-468 2014-07-01 16:52:29,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: finishing write tid 6 to mira0437250-466 2014-07-01 16:52:29,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 82 (ObjNum 465 snap 0 seq_num 465) dirty exists 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: finishing write tid 4 to mira0437250-468 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 468 v 76 (ObjNum 467 snap 0 seq_num 467) dirty exists 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 1 to mira0437250-469 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 3 to mira0437250-469 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: finishing write tid 4 to mira0437250-469 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 89 (ObjNum 468 snap 0 seq_num 468) dirty exists 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:466: done (15 left) 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:468: done (14 left) 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:469: done (13 left) 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: write initial oid 474 2014-07-01 16:52:29,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: seq_num 473 ranges {453461=603581,1615986=749632,2795614=462535,3653460=1} 2014-07-01 16:52:29,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: writing mira0437250-474 from 453461 to 1057042 tid 1 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: finishing write tid 4 to mira0435601-155 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: finishing write tid 5 to mira0435601-155 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 261 (ObjNum 1565 snap 0 seq_num 1565) dirty exists 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 1 to mira0435601-232 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: done (15 left) 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2428: done (14 left) 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2445: read oid 107 snap -1 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 2 to mira0435601-232 2014-07-01 16:52:29,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: expect (ObjNum 889 snap 0 seq_num 889) 2014-07-01 16:52:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: writing mira0437250-474 from 1615986 to 2365618 tid 2 2014-07-01 16:52:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: writing mira0437250-474 from 2795614 to 3258149 tid 3 2014-07-01 16:52:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: writing mira0437250-474 from 3653460 to 3653461 tid 4 2014-07-01 16:52:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: write initial oid 475 2014-07-01 16:52:29,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: seq_num 474 ranges {665043=459033,1576646=419827,2443367=748302,3799992=65051} 2014-07-01 16:52:29,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: writing mira0437250-475 from 665043 to 1124076 tid 1 2014-07-01 16:52:29,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: writing mira0437250-475 from 1576646 to 1996473 tid 2 2014-07-01 16:52:29,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: writing mira0437250-475 from 2443367 to 3191669 tid 3 2014-07-01 16:52:29,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: writing mira0437250-475 from 3799992 to 3865043 tid 4 2014-07-01 16:52:29,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: write initial oid 476 2014-07-01 16:52:29,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: seq_num 475 ranges {608848=1} 2014-07-01 16:52:29,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: writing mira0437250-476 from 608848 to 608849 tid 1 2014-07-01 16:52:29,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 1 to mira0437250-472 2014-07-01 16:52:29,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 2 to mira0437250-472 2014-07-01 16:52:29,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 4 to mira0437250-472 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: finishing write tid 5 to mira0437250-472 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 79 (ObjNum 471 snap 0 seq_num 471) dirty exists 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 1 to mira0437250-473 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:472: done (15 left) 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: write initial oid 477 2014-07-01 16:52:29,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: seq_num 476 ranges {553679=555090,1353678=1} 2014-07-01 16:52:29,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: writing mira0437250-477 from 553679 to 1108769 tid 1 2014-07-01 16:52:29,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: writing mira0437250-477 from 1353678 to 1353679 tid 2 2014-07-01 16:52:29,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 2 to mira0437250-473 2014-07-01 16:52:29,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 3 to mira0437250-473 2014-07-01 16:52:29,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 5 to mira0437250-473 2014-07-01 16:52:29,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: finishing write tid 6 to mira0437250-473 2014-07-01 16:52:29,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 86 (ObjNum 472 snap 0 seq_num 472) dirty exists 2014-07-01 16:52:29,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 1 to mira0437250-474 2014-07-01 16:52:29,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 2 to mira0437250-474 2014-07-01 16:52:29,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 3 to mira0437250-474 2014-07-01 16:52:29,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 4 to mira0437250-474 2014-07-01 16:52:29,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 6 to mira0437250-474 2014-07-01 16:52:29,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: finishing write tid 7 to mira0437250-474 2014-07-01 16:52:29,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 84 (ObjNum 473 snap 0 seq_num 473) dirty exists 2014-07-01 16:52:29,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 1 to mira0437250-475 2014-07-01 16:52:29,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 2 to mira0437250-475 2014-07-01 16:52:29,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 3 to mira0437250-475 2014-07-01 16:52:29,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 4 to mira0437250-475 2014-07-01 16:52:29,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 6 to mira0437250-475 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: finishing write tid 7 to mira0437250-475 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 95 (ObjNum 474 snap 0 seq_num 474) dirty exists 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:473: done (15 left) 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:474: done (14 left) 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:475: done (13 left) 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: write initial oid 478 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: seq_num 477 ranges {648908=400000} 2014-07-01 16:52:29,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: writing mira0437250-478 from 648908 to 1048908 tid 1 2014-07-01 16:52:29,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: write initial oid 479 2014-07-01 16:52:29,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: seq_num 478 ranges {710090=534795,1963031=347059} 2014-07-01 16:52:29,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0437250-479 from 710090 to 1244885 tid 1 2014-07-01 16:52:29,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: writing mira0437250-479 from 1963031 to 2310090 tid 2 2014-07-01 16:52:29,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: write initial oid 480 2014-07-01 16:52:29,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: seq_num 479 ranges {406344=658023,1828587=177757} 2014-07-01 16:52:29,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: writing mira0437250-480 from 406344 to 1064367 tid 1 2014-07-01 16:52:29,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: writing mira0437250-480 from 1828587 to 2006344 tid 2 2014-07-01 16:52:29,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 1 to mira0437250-477 2014-07-01 16:52:29,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 2 to mira0437250-477 2014-07-01 16:52:29,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 4 to mira0437250-477 2014-07-01 16:52:29,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: finishing write tid 5 to mira0437250-477 2014-07-01 16:52:29,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 118 (ObjNum 476 snap 0 seq_num 476) dirty exists 2014-07-01 16:52:29,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 1 to mira0437250-478 2014-07-01 16:52:29,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 3 to mira0437250-478 2014-07-01 16:52:29,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: finishing write tid 4 to mira0437250-478 2014-07-01 16:52:29,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 478 v 91 (ObjNum 477 snap 0 seq_num 477) dirty exists 2014-07-01 16:52:29,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 1 to mira0437250-479 2014-07-01 16:52:29,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 2 to mira0437250-479 2014-07-01 16:52:29,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 4 to mira0437250-479 2014-07-01 16:52:29,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: finishing write tid 5 to mira0437250-479 2014-07-01 16:52:29,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 78 (ObjNum 478 snap 0 seq_num 478) dirty exists 2014-07-01 16:52:29,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:477: done (15 left) 2014-07-01 16:52:29,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:478: done (14 left) 2014-07-01 16:52:29,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:479: done (13 left) 2014-07-01 16:52:29,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: write initial oid 481 2014-07-01 16:52:29,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: seq_num 480 ranges {750149=630852,2156223=593926} 2014-07-01 16:52:29,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: writing mira0437250-481 from 750149 to 1381001 tid 1 2014-07-01 16:52:29,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: writing mira0437250-481 from 2156223 to 2750149 tid 2 2014-07-01 16:52:29,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: write initial oid 482 2014-07-01 16:52:29,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: seq_num 481 ranges {694979=602656,1718931=697690,2849321=645658} 2014-07-01 16:52:29,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: writing mira0437250-482 from 694979 to 1297635 tid 1 2014-07-01 16:52:29,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: writing mira0437250-482 from 1718931 to 2416621 tid 2 2014-07-01 16:52:29,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: writing mira0437250-482 from 2849321 to 3494979 tid 3 2014-07-01 16:52:29,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: write initial oid 483 2014-07-01 16:52:29,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: seq_num 482 ranges {790209=458108,1831016=519309,2834995=355214} 2014-07-01 16:52:29,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0437250-483 from 790209 to 1248317 tid 1 2014-07-01 16:52:29,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0437250-483 from 1831016 to 2350325 tid 2 2014-07-01 16:52:29,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: writing mira0437250-483 from 2834995 to 3190209 tid 3 2014-07-01 16:52:29,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 1 to mira0437250-480 2014-07-01 16:52:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 2 to mira0437250-480 2014-07-01 16:52:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 4 to mira0437250-480 2014-07-01 16:52:29,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: finishing write tid 5 to mira0437250-480 2014-07-01 16:52:29,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 98 (ObjNum 479 snap 0 seq_num 479) dirty exists 2014-07-01 16:52:29,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 1 to mira0437250-482 2014-07-01 16:52:29,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 2 to mira0437250-482 2014-07-01 16:52:29,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 3 to mira0437250-482 2014-07-01 16:52:29,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 5 to mira0437250-482 2014-07-01 16:52:29,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: finishing write tid 6 to mira0437250-482 2014-07-01 16:52:29,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 102 (ObjNum 481 snap 0 seq_num 481) dirty exists 2014-07-01 16:52:29,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 1 to mira0437250-483 2014-07-01 16:52:29,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:480: done (15 left) 2014-07-01 16:52:29,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:482: done (14 left) 2014-07-01 16:52:29,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: write initial oid 484 2014-07-01 16:52:29,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: seq_num 483 ranges {450366=1} 2014-07-01 16:52:29,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: writing mira0437250-484 from 450366 to 450367 tid 1 2014-07-01 16:52:29,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: write initial oid 485 2014-07-01 16:52:29,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: seq_num 484 ranges {795196=400000} 2014-07-01 16:52:29,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: writing mira0437250-485 from 795196 to 1195196 tid 1 2014-07-01 16:52:29,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 2 to mira0437250-483 2014-07-01 16:52:29,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 3 to mira0437250-483 2014-07-01 16:52:29,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 1 to mira0437250-484 2014-07-01 16:52:29,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 5 to mira0437250-483 2014-07-01 16:52:29,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 3 to mira0437250-484 2014-07-01 16:52:29,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: finishing write tid 4 to mira0437250-484 2014-07-01 16:52:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 88 (ObjNum 483 snap 0 seq_num 483) dirty exists 2014-07-01 16:52:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: finishing write tid 6 to mira0437250-483 2014-07-01 16:52:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 121 (ObjNum 482 snap 0 seq_num 482) dirty exists 2014-07-01 16:52:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:483: done (15 left) 2014-07-01 16:52:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:484: done (14 left) 2014-07-01 16:52:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: write initial oid 486 2014-07-01 16:52:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: seq_num 485 ranges {490426=400000} 2014-07-01 16:52:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: writing mira0437250-486 from 490426 to 890426 tid 1 2014-07-01 16:52:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: write initial oid 487 2014-07-01 16:52:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: seq_num 486 ranges {435256=532846,1635255=1} 2014-07-01 16:52:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: writing mira0437250-487 from 435256 to 968102 tid 1 2014-07-01 16:52:29,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: writing mira0437250-487 from 1635255 to 1635256 tid 2 2014-07-01 16:52:29,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:29,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 1 to mira0437250-485 2014-07-01 16:52:29,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 3 to mira0437250-485 2014-07-01 16:52:29,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: finishing write tid 4 to mira0437250-485 2014-07-01 16:52:29,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 104 (ObjNum 484 snap 0 seq_num 484) dirty exists 2014-07-01 16:52:29,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 1 to mira0437250-486 2014-07-01 16:52:29,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 3 to mira0437250-486 2014-07-01 16:52:29,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: finishing write tid 4 to mira0437250-486 2014-07-01 16:52:29,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 84 (ObjNum 485 snap 0 seq_num 485) dirty exists 2014-07-01 16:52:29,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:485: done (15 left) 2014-07-01 16:52:29,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:486: done (14 left) 2014-07-01 16:52:29,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: write initial oid 488 2014-07-01 16:52:29,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: seq_num 487 ranges {495414=657098,1895437=656781,2895413=1} 2014-07-01 16:52:29,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0437250-488 from 495414 to 1152512 tid 1 2014-07-01 16:52:29,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0437250-488 from 1895437 to 2552218 tid 2 2014-07-01 16:52:29,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: writing mira0437250-488 from 2895413 to 2895414 tid 3 2014-07-01 16:52:29,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: write initial oid 489 2014-07-01 16:52:29,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: seq_num 488 ranges {591667=628902,2009568=582099} 2014-07-01 16:52:29,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: writing mira0437250-489 from 591667 to 1220569 tid 1 2014-07-01 16:52:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: writing mira0437250-489 from 2009568 to 2591667 tid 2 2014-07-01 16:52:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 1 to mira0437250-487 2014-07-01 16:52:30,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 2 to mira0437250-487 2014-07-01 16:52:30,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 4 to mira0437250-487 2014-07-01 16:52:30,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: finishing write tid 5 to mira0437250-487 2014-07-01 16:52:30,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 124 (ObjNum 486 snap 0 seq_num 486) dirty exists 2014-07-01 16:52:30,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 1 to mira0437250-488 2014-07-01 16:52:30,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 2 to mira0437250-488 2014-07-01 16:52:30,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 3 to mira0437250-488 2014-07-01 16:52:30,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 5 to mira0437250-488 2014-07-01 16:52:30,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: finishing write tid 6 to mira0437250-488 2014-07-01 16:52:30,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 92 (ObjNum 487 snap 0 seq_num 487) dirty exists 2014-07-01 16:52:30,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 1 to mira0437250-489 2014-07-01 16:52:30,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:487: done (15 left) 2014-07-01 16:52:30,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:488: done (14 left) 2014-07-01 16:52:30,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: write initial oid 490 2014-07-01 16:52:30,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: seq_num 489 ranges {687921=752131,2123701=164220} 2014-07-01 16:52:30,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: writing mira0437250-490 from 687921 to 1440052 tid 1 2014-07-01 16:52:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: writing mira0437250-490 from 2123701 to 2287921 tid 2 2014-07-01 16:52:30,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: write initial oid 491 2014-07-01 16:52:30,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: seq_num 490 ranges {80302=1} 2014-07-01 16:52:30,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: writing mira0437250-491 from 80302 to 80303 tid 1 2014-07-01 16:52:30,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 2 to mira0437250-489 2014-07-01 16:52:30,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 4 to mira0437250-489 2014-07-01 16:52:30,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: finishing write tid 5 to mira0437250-489 2014-07-01 16:52:30,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 121 (ObjNum 488 snap 0 seq_num 488) dirty exists 2014-07-01 16:52:30,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:489: done (15 left) 2014-07-01 16:52:30,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: write initial oid 492 2014-07-01 16:52:30,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: seq_num 491 ranges {575533=731835,2048093=440410,3276424=499109} 2014-07-01 16:52:30,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0437250-492 from 575533 to 1307368 tid 1 2014-07-01 16:52:30,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0437250-492 from 2048093 to 2488503 tid 2 2014-07-01 16:52:30,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: writing mira0437250-492 from 3276424 to 3775533 tid 3 2014-07-01 16:52:30,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 1 to mira0437250-491 2014-07-01 16:52:30,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 3 to mira0437250-491 2014-07-01 16:52:30,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: finishing write tid 4 to mira0437250-491 2014-07-01 16:52:30,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 80 (ObjNum 490 snap 0 seq_num 490) dirty exists 2014-07-01 16:52:30,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 1 to mira0437250-492 2014-07-01 16:52:30,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 2 to mira0437250-492 2014-07-01 16:52:30,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:491: done (15 left) 2014-07-01 16:52:30,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: write initial oid 493 2014-07-01 16:52:30,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: seq_num 492 ranges {787114=552216,1842481=757133,3254033=514179,3987113=1} 2014-07-01 16:52:30,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0437250-493 from 787114 to 1339330 tid 1 2014-07-01 16:52:30,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0437250-493 from 1842481 to 2599614 tid 2 2014-07-01 16:52:30,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0437250-493 from 3254033 to 3768212 tid 3 2014-07-01 16:52:30,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: writing mira0437250-493 from 3987113 to 3987114 tid 4 2014-07-01 16:52:30,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 3 to mira0437250-492 2014-07-01 16:52:30,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 5 to mira0437250-492 2014-07-01 16:52:30,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: finishing write tid 6 to mira0437250-492 2014-07-01 16:52:30,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 108 (ObjNum 491 snap 0 seq_num 491) dirty exists 2014-07-01 16:52:30,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 1 to mira0437250-493 2014-07-01 16:52:30,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 2 to mira0437250-493 2014-07-01 16:52:30,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:492: done (15 left) 2014-07-01 16:52:30,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: write initial oid 494 2014-07-01 16:52:30,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: seq_num 493 ranges {580520=675444,1780519=1} 2014-07-01 16:52:30,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0437250-494 from 580520 to 1255964 tid 1 2014-07-01 16:52:30,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: writing mira0437250-494 from 1780519 to 1780520 tid 2 2014-07-01 16:52:30,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 3 to mira0437250-493 2014-07-01 16:52:30,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 4 to mira0437250-493 2014-07-01 16:52:30,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 6 to mira0437250-493 2014-07-01 16:52:30,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: finishing write tid 7 to mira0437250-493 2014-07-01 16:52:30,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 129 (ObjNum 492 snap 0 seq_num 492) dirty exists 2014-07-01 16:52:30,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:493: done (15 left) 2014-07-01 16:52:30,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: write initial oid 495 2014-07-01 16:52:30,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: seq_num 494 ranges {676774=530896,1476773=1} 2014-07-01 16:52:30,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: writing mira0437250-495 from 676774 to 1207670 tid 1 2014-07-01 16:52:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: writing mira0437250-495 from 1476773 to 1476774 tid 2 2014-07-01 16:52:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 1 to mira0437250-494 2014-07-01 16:52:30,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 2 to mira0437250-494 2014-07-01 16:52:30,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 4 to mira0437250-494 2014-07-01 16:52:30,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: finishing write tid 5 to mira0437250-494 2014-07-01 16:52:30,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 83 (ObjNum 493 snap 0 seq_num 493) dirty exists 2014-07-01 16:52:30,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:494: done (15 left) 2014-07-01 16:52:30,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: write initial oid 496 2014-07-01 16:52:30,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: seq_num 495 ranges {772004=400000} 2014-07-01 16:52:30,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: writing mira0437250-496 from 772004 to 1172004 tid 1 2014-07-01 16:52:30,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 1 to mira0437250-495 2014-07-01 16:52:30,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 2 to mira0437250-495 2014-07-01 16:52:30,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 4 to mira0437250-495 2014-07-01 16:52:30,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: finishing write tid 5 to mira0437250-495 2014-07-01 16:52:30,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 86 (ObjNum 494 snap 0 seq_num 494) dirty exists 2014-07-01 16:52:30,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:495: done (15 left) 2014-07-01 16:52:30,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: write initial oid 497 2014-07-01 16:52:30,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: seq_num 496 ranges {681761=626953,1809665=424410,2690358=687646,3481760=1} 2014-07-01 16:52:30,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0437250-497 from 681761 to 1308714 tid 1 2014-07-01 16:52:30,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0437250-497 from 1809665 to 2234075 tid 2 2014-07-01 16:52:30,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0437250-497 from 2690358 to 3378004 tid 3 2014-07-01 16:52:30,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: writing mira0437250-497 from 3481760 to 3481761 tid 4 2014-07-01 16:52:30,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 1 to mira0437250-496 2014-07-01 16:52:30,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 3 to mira0437250-496 2014-07-01 16:52:30,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: finishing write tid 4 to mira0437250-496 2014-07-01 16:52:30,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 90 (ObjNum 495 snap 0 seq_num 495) dirty exists 2014-07-01 16:52:30,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:496: done (15 left) 2014-07-01 16:52:30,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: write initial oid 498 2014-07-01 16:52:30,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: seq_num 497 ranges {776991=750181,2189525=646029,3176990=1} 2014-07-01 16:52:30,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: writing mira0437250-498 from 776991 to 1527172 tid 1 2014-07-01 16:52:30,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: writing mira0437250-498 from 2189525 to 2835554 tid 2 2014-07-01 16:52:30,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: writing mira0437250-498 from 3176990 to 3176991 tid 3 2014-07-01 16:52:30,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 2 to mira0437250-462 2014-07-01 16:52:30,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 4 to mira0437250-462 2014-07-01 16:52:30,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: finishing write tid 5 to mira0437250-462 2014-07-01 16:52:30,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 88 (ObjNum 461 snap 0 seq_num 461) dirty exists 2014-07-01 16:52:30,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:462: done (15 left) 2014-07-01 16:52:30,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: write initial oid 499 2014-07-01 16:52:30,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: seq_num 498 ranges {473245=606658,1788330=716224,2873244=1} 2014-07-01 16:52:30,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0437250-499 from 473245 to 1079903 tid 1 2014-07-01 16:52:30,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0437250-499 from 1788330 to 2504554 tid 2 2014-07-01 16:52:30,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: writing mira0437250-499 from 2873244 to 2873245 tid 3 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 1 to mira0437250-463 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 2 to mira0437250-463 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 4 to mira0437250-463 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 1 to mira0437250-467 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: finishing write tid 5 to mira0437250-463 2014-07-01 16:52:30,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 120 (ObjNum 462 snap 0 seq_num 462) dirty exists 2014-07-01 16:52:30,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 3 to mira0437250-467 2014-07-01 16:52:30,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: finishing write tid 4 to mira0437250-467 2014-07-01 16:52:30,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 122 (ObjNum 466 snap 0 seq_num 466) dirty exists 2014-07-01 16:52:30,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 1 to mira0437250-470 2014-07-01 16:52:30,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:463: done (15 left) 2014-07-01 16:52:30,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:467: done (14 left) 2014-07-01 16:52:30,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: write initial oid 500 2014-07-01 16:52:30,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: seq_num 499 ranges {665625=1} 2014-07-01 16:52:30,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: writing mira0437250-500 from 665625 to 665626 tid 1 2014-07-01 16:52:30,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: rollback oid 326 current snap is 0 2014-07-01 16:52:30,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:501: done (15 left) 2014-07-01 16:52:30,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:502: snap_create 2014-07-01 16:52:30,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 2 to mira0437250-470 2014-07-01 16:52:30,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 4 to mira0437250-470 2014-07-01 16:52:30,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: finishing write tid 5 to mira0437250-470 2014-07-01 16:52:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 91 (ObjNum 469 snap 0 seq_num 469) dirty exists 2014-07-01 16:52:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 1 to mira0437250-476 2014-07-01 16:52:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 1 to mira0437250-481 2014-07-01 16:52:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: finishing write tid 4 to mira0437250-459 2014-07-01 16:52:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 115 (ObjNum 458 snap 0 seq_num 458) dirty exists 2014-07-01 16:52:30,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: finishing write tid 6 to mira0437250-449 2014-07-01 16:52:30,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 113 (ObjNum 448 snap 0 seq_num 448) dirty exists 2014-07-01 16:52:30,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 3 to mira0437250-476 2014-07-01 16:52:30,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 2 to mira0437250-481 2014-07-01 16:52:30,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: finishing write tid 4 to mira0437250-476 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 124 (ObjNum 475 snap 0 seq_num 475) dirty exists 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: finishing write tid 5 to mira0437250-447 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 85 (ObjNum 446 snap 0 seq_num 446) dirty exists 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 4 to mira0437250-481 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: finishing write tid 5 to mira0437250-445 2014-07-01 16:52:30,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 108 (ObjNum 444 snap 0 seq_num 444) dirty exists 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: finishing write tid 4 to mira0437250-444 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 109 (ObjNum 443 snap 0 seq_num 443) dirty exists 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: finishing write tid 5 to mira0437250-481 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 94 (ObjNum 480 snap 0 seq_num 480) dirty exists 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 1 to mira0437250-497 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 2 to mira0437250-497 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 3 to mira0437250-497 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 4 to mira0437250-497 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 6 to mira0437250-497 2014-07-01 16:52:30,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: finishing write tid 7 to mira0437250-497 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 129 (ObjNum 496 snap 0 seq_num 496) dirty exists 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 1 to mira0437250-465 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 2 to mira0437250-465 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 3 to mira0437250-465 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 5 to mira0437250-465 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 1 to mira0437250-471 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 2 to mira0437250-471 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 4 to mira0437250-471 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 1 to mira0437250-490 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 2 to mira0437250-490 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 4 to mira0437250-490 2014-07-01 16:52:30,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 1 to mira0437250-498 2014-07-01 16:52:30,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 2 to mira0437250-498 2014-07-01 16:52:30,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 1 to mira0437250-499 2014-07-01 16:52:30,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 3 to mira0437250-498 2014-07-01 16:52:30,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 5 to mira0437250-498 2014-07-01 16:52:30,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 2 to mira0437250-499 2014-07-01 16:52:30,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 3 to mira0437250-499 2014-07-01 16:52:30,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 5 to mira0437250-499 2014-07-01 16:52:30,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 1 to mira0437250-500 2014-07-01 16:52:30,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 3 to mira0437250-500 2014-07-01 16:52:30,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: finishing write tid 4 to mira0437250-500 2014-07-01 16:52:30,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 500 v 92 (ObjNum 499 snap 0 seq_num 499) dirty exists 2014-07-01 16:52:30,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: finishing write tid 6 to mira0437250-499 2014-07-01 16:52:30,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 499 v 133 (ObjNum 498 snap 0 seq_num 498) dirty exists 2014-07-01 16:52:30,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: finishing write tid 6 to mira0437250-465 2014-07-01 16:52:30,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 465 v 119 (ObjNum 464 snap 0 seq_num 464) dirty exists 2014-07-01 16:52:30,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: finishing write tid 5 to mira0437250-471 2014-07-01 16:52:30,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 122 (ObjNum 470 snap 0 seq_num 470) dirty exists 2014-07-01 16:52:30,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: finishing write tid 5 to mira0437250-490 2014-07-01 16:52:30,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 111 (ObjNum 489 snap 0 seq_num 489) dirty exists 2014-07-01 16:52:30,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: finishing write tid 6 to mira0437250-498 2014-07-01 16:52:30,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 115 (ObjNum 497 snap 0 seq_num 497) dirty exists 2014-07-01 16:52:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:444: done (15 left) 2014-07-01 16:52:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:445: done (14 left) 2014-07-01 16:52:30,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:447: done (13 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:449: done (12 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:459: done (11 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:465: done (10 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:470: done (9 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:471: done (8 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:476: done (7 left) 2014-07-01 16:52:30,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:481: done (6 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:490: done (5 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:497: done (4 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:498: done (3 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:499: done (2 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:500: done (1 left) 2014-07-01 16:52:30,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:502: done (0 left) 2014-07-01 16:52:30,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: read oid 336 snap -1 2014-07-01 16:52:30,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: write oid 43 current snap is 1 2014-07-01 16:52:30,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: seq_num 500 ranges {360855=1} 2014-07-01 16:52:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: writing mira0437250-43 from 360855 to 360856 tid 1 2014-07-01 16:52:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: rollback oid 98 current snap is 1 2014-07-01 16:52:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 98 to 0 2014-07-01 16:52:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: write oid 59 current snap is 1 2014-07-01 16:52:30,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: seq_num 501 ranges {573461=1} 2014-07-01 16:52:30,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: writing mira0437250-59 from 573461 to 573462 tid 1 2014-07-01 16:52:30,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: read oid 292 snap -1 2014-07-01 16:52:30,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: read oid 438 snap -1 2014-07-01 16:52:30,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: delete oid 443 current snap is 1 2014-07-01 16:52:30,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 1 to mira0437250-43 2014-07-01 16:52:30,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 1 to mira0437250-59 2014-07-01 16:52:30,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 3 to mira0437250-59 2014-07-01 16:52:30,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 3 to mira0437250-43 2014-07-01 16:52:30,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: finishing write tid 4 to mira0437250-43 2014-07-01 16:52:30,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 118 (ObjNum 500 snap 1 seq_num 500) dirty exists 2014-07-01 16:52:30,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: expect (ObjNum 437 snap 0 seq_num 437) 2014-07-01 16:52:31,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 3 to mira0435601-232 2014-07-01 16:52:31,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 4 to mira0435601-232 2014-07-01 16:52:31,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 6 to mira0435601-232 2014-07-01 16:52:31,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: expect (ObjNum 1403 snap 0 seq_num 1403) 2014-07-01 16:52:31,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: expect (ObjNum 335 snap 0 seq_num 335) 2014-07-01 16:52:32,190.190 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:32,191.191 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:32,191.191 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:52:32,191.191 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:52:32,191.191 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:52:32,192.192 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:52:32,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 1 to mira0435601-56 2014-07-01 16:52:32,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 2 to mira0435601-56 2014-07-01 16:52:32,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 3 to mira0435601-56 2014-07-01 16:52:32,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 4 to mira0435601-56 2014-07-01 16:52:32,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 6 to mira0435601-56 2014-07-01 16:52:32,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: finishing write tid 1 to mira0435601-97 2014-07-01 16:52:32,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: finishing write tid 2 to mira0435601-97 2014-07-01 16:52:32,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: done (14 left) 2014-07-01 16:52:32,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: done (13 left) 2014-07-01 16:52:32,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: delete oid 290 current snap is 0 2014-07-01 16:52:32,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: finishing write tid 3 to mira0435601-97 2014-07-01 16:52:32,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: finishing write tid 5 to mira0435601-97 2014-07-01 16:52:32,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 1 to mira0435601-151 2014-07-01 16:52:32,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 2 to mira0435601-151 2014-07-01 16:52:32,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 3 to mira0435601-151 2014-07-01 16:52:32,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 5 to mira0435601-151 2014-07-01 16:52:32,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: finishing write tid 1 to mira0435601-127 2014-07-01 16:52:32,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: finishing write tid 3 to mira0435601-127 2014-07-01 16:52:32,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: finishing write tid 6 to mira0435601-97 2014-07-01 16:52:32,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 264 (ObjNum 1570 snap 0 seq_num 1570) dirty exists 2014-07-01 16:52:32,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: finishing write tid 4 to mira0435601-127 2014-07-01 16:52:32,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 190 (ObjNum 1573 snap 0 seq_num 1573) dirty exists 2014-07-01 16:52:32,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 6 to mira0435601-151 2014-07-01 16:52:32,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 272 (ObjNum 1571 snap 0 seq_num 1571) dirty exists 2014-07-01 16:52:32,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: expect (ObjNum 1489 snap 0 seq_num 1489) 2014-07-01 16:52:33,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: expect (ObjNum 291 snap 0 seq_num 291) 2014-07-01 16:52:33,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 88 (ObjNum 97 snap 0 seq_num 97) dirty exists 2014-07-01 16:52:33,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: finishing write tid 4 to mira0437250-59 2014-07-01 16:52:33,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 86 (ObjNum 501 snap 1 seq_num 501) dirty exists 2014-07-01 16:52:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:503: done (6 left) 2014-07-01 16:52:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:504: done (5 left) 2014-07-01 16:52:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:505: done (4 left) 2014-07-01 16:52:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:506: done (3 left) 2014-07-01 16:52:33,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:507: done (2 left) 2014-07-01 16:52:33,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:508: done (1 left) 2014-07-01 16:52:33,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:509: done (0 left) 2014-07-01 16:52:33,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: snap_create 2014-07-01 16:52:34,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:510: done (0 left) 2014-07-01 16:52:34,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: snap_create 2014-07-01 16:52:34,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 1 to mira0435601-14 2014-07-01 16:52:34,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 2 to mira0435601-14 2014-07-01 16:52:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 3 to mira0435601-14 2014-07-01 16:52:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2441: expect (ObjNum 1475 snap 0 seq_num 1475) 2014-07-01 16:52:35,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: finishing write tid 7 to mira0435601-232 2014-07-01 16:52:35,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 117 (ObjNum 1566 snap 0 seq_num 1566) dirty exists 2014-07-01 16:52:35,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 4 to mira0435601-14 2014-07-01 16:52:35,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 6 to mira0435601-14 2014-07-01 16:52:35,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: finishing write tid 7 to mira0435601-14 2014-07-01 16:52:35,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 144 (ObjNum 1569 snap 0 seq_num 1569) dirty exists 2014-07-01 16:52:35,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:2437: expect (ObjNum 1466 snap 0 seq_num 1466) 2014-07-01 16:52:35,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:511: done (0 left) 2014-07-01 16:52:35,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: write oid 453 current snap is 3 2014-07-01 16:52:35,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: seq_num 503 ranges {462097=653199,1583950=418624,2062096=1} 2014-07-01 16:52:35,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: writing mira0437250-453 from 462097 to 1115296 tid 1 2014-07-01 16:52:35,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: writing mira0437250-453 from 1583950 to 2002574 tid 2 2014-07-01 16:52:35,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: writing mira0437250-453 from 2062096 to 2062097 tid 3 2014-07-01 16:52:35,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: read oid 44 snap -1 2014-07-01 16:52:35,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:514: delete oid 499 current snap is 3 2014-07-01 16:52:35,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: finishing write tid 1 to mira0437250-453 2014-07-01 16:52:35,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: finishing write tid 2 to mira0437250-453 2014-07-01 16:52:35,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: finishing write tid 3 to mira0437250-453 2014-07-01 16:52:35,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: finishing write tid 5 to mira0437250-453 2014-07-01 16:52:35,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: expect (ObjNum 43 snap 0 seq_num 43) 2014-07-01 16:52:36,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:513: done (2 left) 2014-07-01 16:52:36,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:514: done (1 left) 2014-07-01 16:52:36,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: read oid 263 snap 1 2014-07-01 16:52:36,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:516: rollback oid 465 current snap is 3 2014-07-01 16:52:36,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 465 to 2 2014-07-01 16:52:36,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: rollback oid 299 current snap is 3 2014-07-01 16:52:36,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 299 to 1 2014-07-01 16:52:36,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: read oid 117 snap 0 2014-07-01 16:52:36,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: read oid 103 snap 1 2014-07-01 16:52:36,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: write oid 275 current snap is 3 2014-07-01 16:52:36,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: seq_num 505 ranges {618509=750280,1893495=594086,3018508=1} 2014-07-01 16:52:36,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: writing mira0437250-275 from 618509 to 1368789 tid 1 2014-07-01 16:52:36,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: writing mira0437250-275 from 1893495 to 2487581 tid 2 2014-07-01 16:52:36,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: writing mira0437250-275 from 3018508 to 3018509 tid 3 2014-07-01 16:52:36,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: write oid 125 current snap is 3 2014-07-01 16:52:36,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: seq_num 506 ranges {563339=605732,1739851=664282,2815637=638815,3763338=1} 2014-07-01 16:52:36,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: writing mira0437250-125 from 563339 to 1169071 tid 1 2014-07-01 16:52:36,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: writing mira0437250-125 from 1739851 to 2404133 tid 2 2014-07-01 16:52:36,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: writing mira0437250-125 from 2815637 to 3454452 tid 3 2014-07-01 16:52:36,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: writing mira0437250-125 from 3763338 to 3763339 tid 4 2014-07-01 16:52:36,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: write oid 321 current snap is 3 2014-07-01 16:52:36,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: seq_num 507 ranges {506119=1} 2014-07-01 16:52:36,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: writing mira0437250-321 from 506119 to 506120 tid 1 2014-07-01 16:52:36,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:523: snap_create 2014-07-01 16:52:36,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: expect (ObjNum 116 snap 0 seq_num 116) 2014-07-01 16:52:36,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: expect (ObjNum 262 snap 0 seq_num 262) 2014-07-01 16:52:36,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 465 v 124 (ObjNum 464 snap 0 seq_num 464) dirty exists 2014-07-01 16:52:36,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: expect (ObjNum 102 snap 0 seq_num 102) 2014-07-01 16:52:36,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 96 (ObjNum 298 snap 0 seq_num 298) dirty exists 2014-07-01 16:52:36,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: finishing write tid 1 to mira0437250-275 2014-07-01 16:52:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: finishing write tid 2 to mira0437250-275 2014-07-01 16:52:36,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: finishing write tid 3 to mira0437250-275 2014-07-01 16:52:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: finishing write tid 5 to mira0437250-275 2014-07-01 16:52:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: finishing write tid 6 to mira0437250-275 2014-07-01 16:52:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 275 v 123 (ObjNum 505 snap 3 seq_num 505) dirty exists 2014-07-01 16:52:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 1 to mira0437250-125 2014-07-01 16:52:36,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 2 to mira0437250-125 2014-07-01 16:52:36,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: finishing write tid 1 to mira0437250-321 2014-07-01 16:52:36,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: finishing write tid 3 to mira0437250-321 2014-07-01 16:52:36,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 3 to mira0437250-125 2014-07-01 16:52:36,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 4 to mira0437250-125 2014-07-01 16:52:36,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 6 to mira0437250-125 2014-07-01 16:52:36,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: finishing write tid 7 to mira0437250-125 2014-07-01 16:52:36,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 91 (ObjNum 506 snap 3 seq_num 506) dirty exists 2014-07-01 16:52:36,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: finishing write tid 4 to mira0437250-321 2014-07-01 16:52:36,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 126 (ObjNum 507 snap 3 seq_num 507) dirty exists 2014-07-01 16:52:36,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:515: done (9 left) 2014-07-01 16:52:36,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:516: done (8 left) 2014-07-01 16:52:36,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:517: done (7 left) 2014-07-01 16:52:36,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:518: done (6 left) 2014-07-01 16:52:36,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:519: done (5 left) 2014-07-01 16:52:36,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:520: done (4 left) 2014-07-01 16:52:36,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:521: done (3 left) 2014-07-01 16:52:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:522: done (2 left) 2014-07-01 16:52:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:523: done (1 left) 2014-07-01 16:52:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: write oid 497 current snap is 4 2014-07-01 16:52:36,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: seq_num 508 ranges {202373=1} 2014-07-01 16:52:36,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: writing mira0437250-497 from 202373 to 202374 tid 1 2014-07-01 16:52:36,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:525: rollback oid 183 current snap is 4 2014-07-01 16:52:36,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 183 to 2 2014-07-01 16:52:36,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: write oid 223 current snap is 4 2014-07-01 16:52:36,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: seq_num 509 ranges {547204=400000} 2014-07-01 16:52:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: writing mira0437250-223 from 547204 to 947204 tid 1 2014-07-01 16:52:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: rollback oid 66 current snap is 4 2014-07-01 16:52:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 66 to 2 2014-07-01 16:52:36,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:528: snap_create 2014-07-01 16:52:36,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 86 (ObjNum 182 snap 0 seq_num 182) dirty exists 2014-07-01 16:52:36,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 1 to mira0437250-497 2014-07-01 16:52:37,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 3 to mira0437250-497 2014-07-01 16:52:37,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 97 (ObjNum 65 snap 0 seq_num 65) dirty exists 2014-07-01 16:52:37,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 1 to mira0437250-223 2014-07-01 16:52:37,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 3 to mira0437250-223 2014-07-01 16:52:37,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: finishing write tid 4 to mira0437250-223 2014-07-01 16:52:37,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 94 (ObjNum 509 snap 4 seq_num 509) dirty exists 2014-07-01 16:52:37,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: finishing write tid 4 to mira0437250-497 2014-07-01 16:52:37,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 139 (ObjNum 508 snap 4 seq_num 508) dirty exists 2014-07-01 16:52:37,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2445: expect (ObjNum 106 snap 0 seq_num 106) 2014-07-01 16:52:37,675.675 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:37,676.676 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:37,676.676 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:52:37,677.677 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:52:37,677.677 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:52:37,678.678 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:52:37,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: finishing write tid 6 to mira0437250-453 2014-07-01 16:52:37,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 91 (ObjNum 503 snap 3 seq_num 503) dirty exists 2014-07-01 16:52:37,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: finishing write tid 1 to mira0435601-448 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: finishing write tid 3 to mira0435601-448 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: finishing write tid 7 to mira0435601-347 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 253 (ObjNum 1567 snap 0 seq_num 1567) dirty exists 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: finishing write tid 7 to mira0435601-56 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 120 (ObjNum 1568 snap 0 seq_num 1568) dirty exists 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 4 to mira0435601-200 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 122 (ObjNum 1572 snap 0 seq_num 1572) dirty exists 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: finishing write tid 4 to mira0435601-448 2014-07-01 16:52:37,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 202 (ObjNum 1574 snap 0 seq_num 1574) dirty exists 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: done (13 left) 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: done (12 left) 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: done (11 left) 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: done (10 left) 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2437: done (9 left) 2014-07-01 16:52:37,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: done (8 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: done (7 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: done (6 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2441: done (5 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: done (4 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: done (3 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: done (2 left) 2014-07-01 16:52:37,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2445: done (1 left) 2014-07-01 16:52:37,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: done (0 left) 2014-07-01 16:52:37,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2447: read oid 280 snap -1 2014-07-01 16:52:37,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 240 current snap is 0 2014-07-01 16:52:37,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: seq_num 1576 ranges {2269184=438272,2707456=401408} 2014-07-01 16:52:37,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:512: done (5 left) 2014-07-01 16:52:37,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:524: done (4 left) 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:525: done (3 left) 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:526: done (2 left) 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:527: done (1 left) 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:528: done (0 left) 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: read oid 348 snap -1 2014-07-01 16:52:37,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: write oid 292 current snap is 5 2014-07-01 16:52:38,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: seq_num 510 ranges {607362=680470,1724188=447910,2207361=1} 2014-07-01 16:52:38,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0437250-292 from 607362 to 1287832 tid 1 2014-07-01 16:52:38,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0437250-292 from 1724188 to 2172098 tid 2 2014-07-01 16:52:38,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: writing mira0437250-292 from 2207361 to 2207362 tid 3 2014-07-01 16:52:38,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:531: delete oid 82 current snap is 5 2014-07-01 16:52:38,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: expect (ObjNum 347 snap 0 seq_num 347) 2014-07-01 16:52:38,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: writing mira0435601-240 from 2269184 to 2707456 tid 1 2014-07-01 16:52:38,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: writing mira0435601-240 from 2707456 to 3108864 tid 2 2014-07-01 16:52:38,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2449: delete oid 256 current snap is 0 2014-07-01 16:52:38,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 1 to mira0435601-240 2014-07-01 16:52:38,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2449: done (2 left) 2014-07-01 16:52:38,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: read oid 327 snap -1 2014-07-01 16:52:38,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2451: read oid 100 snap -1 2014-07-01 16:52:38,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 272 current snap is 0 2014-07-01 16:52:38,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: seq_num 1578 ranges {0=630784,630784=495616,1126400=307200} 2014-07-01 16:52:38,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: writing mira0435601-272 from 0 to 630784 tid 1 2014-07-01 16:52:38,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: writing mira0435601-272 from 630784 to 1126400 tid 2 2014-07-01 16:52:38,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: writing mira0435601-272 from 1126400 to 1433600 tid 3 2014-07-01 16:52:38,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:2453: read oid 78 snap -1 2014-07-01 16:52:38,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 2 to mira0435601-240 2014-07-01 16:52:38,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 4 to mira0435601-240 2014-07-01 16:52:38,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 5 to mira0435601-240 2014-07-01 16:52:38,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 143 (ObjNum 1576 snap 0 seq_num 1576) dirty exists 2014-07-01 16:52:38,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:2447: expect (ObjNum 1503 snap 0 seq_num 1503) 2014-07-01 16:52:38,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 1 to mira0437250-292 2014-07-01 16:52:38,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 2 to mira0437250-292 2014-07-01 16:52:38,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 3 to mira0437250-292 2014-07-01 16:52:38,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 5 to mira0437250-292 2014-07-01 16:52:38,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: finishing write tid 6 to mira0437250-292 2014-07-01 16:52:38,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 137 (ObjNum 510 snap 5 seq_num 510) dirty exists 2014-07-01 16:52:38,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:529: done (2 left) 2014-07-01 16:52:38,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:530: done (1 left) 2014-07-01 16:52:38,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:531: done (0 left) 2014-07-01 16:52:38,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: rollback oid 459 current snap is 5 2014-07-01 16:52:38,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 459 to 2 2014-07-01 16:52:38,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: snap_create 2014-07-01 16:52:38,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 126 (ObjNum 458 snap 0 seq_num 458) dirty exists 2014-07-01 16:52:38,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:532: done (1 left) 2014-07-01 16:52:38,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:533: done (0 left) 2014-07-01 16:52:38,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: read oid 430 snap -1 2014-07-01 16:52:38,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: read oid 485 snap -1 2014-07-01 16:52:38,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: read oid 239 snap -1 2014-07-01 16:52:38,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: write oid 323 current snap is 6 2014-07-01 16:52:38,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: seq_num 512 ranges {647421=776527,1917380=587277,2647420=1} 2014-07-01 16:52:38,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: writing mira0437250-323 from 647421 to 1423948 tid 1 2014-07-01 16:52:38,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: writing mira0437250-323 from 1917380 to 2504657 tid 2 2014-07-01 16:52:38,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: writing mira0437250-323 from 2647420 to 2647421 tid 3 2014-07-01 16:52:38,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: expect (ObjNum 238 snap 0 seq_num 238) 2014-07-01 16:52:39,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: expect (ObjNum 429 snap 0 seq_num 429) 2014-07-01 16:52:39,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:52:40,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: finishing write tid 1 to mira0437250-323 2014-07-01 16:52:40,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: finishing write tid 2 to mira0437250-323 2014-07-01 16:52:40,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: finishing write tid 3 to mira0437250-323 2014-07-01 16:52:40,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: finishing write tid 5 to mira0437250-323 2014-07-01 16:52:40,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: finishing write tid 6 to mira0437250-323 2014-07-01 16:52:40,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 96 (ObjNum 512 snap 6 seq_num 512) dirty exists 2014-07-01 16:52:40,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:534: done (3 left) 2014-07-01 16:52:40,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:535: done (2 left) 2014-07-01 16:52:40,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:536: done (1 left) 2014-07-01 16:52:40,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:537: done (0 left) 2014-07-01 16:52:40,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:538: rollback oid 7 current snap is 6 2014-07-01 16:52:40,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 7 to 3 2014-07-01 16:52:40,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: read oid 41 snap -1 2014-07-01 16:52:40,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: read oid 76 snap 1 2014-07-01 16:52:40,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: snap_create 2014-07-01 16:52:40,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 7 v 128 (ObjNum 6 snap 0 seq_num 6) dirty exists 2014-07-01 16:52:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: expect (ObjNum 75 snap 0 seq_num 75) 2014-07-01 16:52:40,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: expect (ObjNum 40 snap 0 seq_num 40) 2014-07-01 16:52:41,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 1 to mira0435601-272 2014-07-01 16:52:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 2 to mira0435601-272 2014-07-01 16:52:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: expect (ObjNum 1323 snap 0 seq_num 1323) 2014-07-01 16:52:41,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:538: done (3 left) 2014-07-01 16:52:41,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:539: done (2 left) 2014-07-01 16:52:41,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:540: done (1 left) 2014-07-01 16:52:41,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:541: done (0 left) 2014-07-01 16:52:41,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:542: snap_create 2014-07-01 16:52:41,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:542: done (0 left) 2014-07-01 16:52:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:543: delete oid 428 current snap is 8 2014-07-01 16:52:41,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:543: done (0 left) 2014-07-01 16:52:41,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: read oid 5 snap -1 2014-07-01 16:52:41,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: read oid 372 snap -1 2014-07-01 16:52:41,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: write oid 118 current snap is 8 2014-07-01 16:52:41,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: seq_num 514 ranges {404857=603783,1441772=762739,2618719=603923,3604856=1} 2014-07-01 16:52:42,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0437250-118 from 404857 to 1008640 tid 1 2014-07-01 16:52:42,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0437250-118 from 1441772 to 2204511 tid 2 2014-07-01 16:52:42,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0437250-118 from 2618719 to 3222642 tid 3 2014-07-01 16:52:42,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: writing mira0437250-118 from 3604856 to 3604857 tid 4 2014-07-01 16:52:42,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: write oid 299 current snap is 8 2014-07-01 16:52:42,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: seq_num 515 ranges {500086=727012,1822656=584358,3121768=178318} 2014-07-01 16:52:42,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: writing mira0437250-299 from 500086 to 1227098 tid 1 2014-07-01 16:52:42,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: writing mira0437250-299 from 1822656 to 2407014 tid 2 2014-07-01 16:52:42,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: writing mira0437250-299 from 3121768 to 3300086 tid 3 2014-07-01 16:52:42,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: snap_create 2014-07-01 16:52:42,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: expect (ObjNum 371 snap 0 seq_num 371) 2014-07-01 16:52:42,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: expect (ObjNum 4 snap 0 seq_num 4) 2014-07-01 16:52:43,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 1 to mira0437250-118 2014-07-01 16:52:43,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 2 to mira0437250-118 2014-07-01 16:52:43,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 3 to mira0437250-118 2014-07-01 16:52:43,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 4 to mira0437250-118 2014-07-01 16:52:43,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 6 to mira0437250-118 2014-07-01 16:52:43,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: finishing write tid 7 to mira0437250-118 2014-07-01 16:52:43,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 100 (ObjNum 514 snap 8 seq_num 514) dirty exists 2014-07-01 16:52:43,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: finishing write tid 1 to mira0437250-299 2014-07-01 16:52:43,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: finishing write tid 2 to mira0437250-299 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: finishing write tid 3 to mira0437250-299 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: finishing write tid 5 to mira0437250-299 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: finishing write tid 6 to mira0437250-299 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 101 (ObjNum 515 snap 8 seq_num 515) dirty exists 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:544: done (4 left) 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:545: done (3 left) 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:546: done (2 left) 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:547: done (1 left) 2014-07-01 16:52:43,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:548: done (0 left) 2014-07-01 16:52:43,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:549: snap_remove snap 4 2014-07-01 16:52:43,153.153 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:43,154.154 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:43,154.154 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 16:52:43,155.155 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:52:43,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 3 to mira0435601-272 2014-07-01 16:52:43,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 5 to mira0435601-272 2014-07-01 16:52:43,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 6 to mira0435601-272 2014-07-01 16:52:43,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 135 (ObjNum 1578 snap 0 seq_num 1578) dirty exists 2014-07-01 16:52:43,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2451: expect (ObjNum 1198 snap 0 seq_num 1198) 2014-07-01 16:52:43,752.752 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:52:43,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:549: done (0 left) 2014-07-01 16:52:43,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: write oid 42 current snap is 9 2014-07-01 16:52:43,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: seq_num 516 ranges {692468=400000} 2014-07-01 16:52:43,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: writing mira0437250-42 from 692468 to 1092468 tid 1 2014-07-01 16:52:43,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: snap_create 2014-07-01 16:52:44,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 1 to mira0437250-42 2014-07-01 16:52:44,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 3 to mira0437250-42 2014-07-01 16:52:44,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: finishing write tid 4 to mira0437250-42 2014-07-01 16:52:44,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 111 (ObjNum 516 snap 9 seq_num 516) dirty exists 2014-07-01 16:52:44,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:550: done (1 left) 2014-07-01 16:52:44,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:551: done (0 left) 2014-07-01 16:52:44,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:552: rollback oid 177 current snap is 10 2014-07-01 16:52:44,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 177 to 6 2014-07-01 16:52:44,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: write oid 498 current snap is 10 2014-07-01 16:52:44,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: seq_num 517 ranges {787697=1} 2014-07-01 16:52:45,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: writing mira0437250-498 from 787697 to 787698 tid 1 2014-07-01 16:52:45,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:554: delete oid 497 current snap is 10 2014-07-01 16:52:45,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 133 (ObjNum 176 snap 0 seq_num 176) dirty exists 2014-07-01 16:52:45,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: finishing write tid 1 to mira0437250-498 2014-07-01 16:52:45,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: finishing write tid 3 to mira0437250-498 2014-07-01 16:52:45,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:552: done (2 left) 2014-07-01 16:52:45,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:554: done (1 left) 2014-07-01 16:52:45,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:555: snap_remove snap 3 2014-07-01 16:52:45,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: finishing write tid 4 to mira0437250-498 2014-07-01 16:52:45,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 136 (ObjNum 517 snap 10 seq_num 517) dirty exists 2014-07-01 16:52:46,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:553: done (1 left) 2014-07-01 16:52:46,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:555: done (0 left) 2014-07-01 16:52:46,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: write oid 396 current snap is 10 2014-07-01 16:52:46,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: seq_num 519 ranges {793709=402773,1789841=465139,2771598=22111} 2014-07-01 16:52:46,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: writing mira0437250-396 from 793709 to 1196482 tid 1 2014-07-01 16:52:46,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: writing mira0437250-396 from 1789841 to 2254980 tid 2 2014-07-01 16:52:46,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: writing mira0437250-396 from 2771598 to 2793709 tid 3 2014-07-01 16:52:46,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:557: snap_remove snap 5 2014-07-01 16:52:46,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: finishing write tid 1 to mira0437250-396 2014-07-01 16:52:46,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: finishing write tid 2 to mira0437250-396 2014-07-01 16:52:46,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: finishing write tid 3 to mira0437250-396 2014-07-01 16:52:46,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: finishing write tid 5 to mira0437250-396 2014-07-01 16:52:46,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: finishing write tid 6 to mira0437250-396 2014-07-01 16:52:46,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 105 (ObjNum 519 snap 10 seq_num 519) dirty exists 2014-07-01 16:52:46,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2453: expect (ObjNum 1392 snap 0 seq_num 1392) 2014-07-01 16:52:46,756.756 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 16:52:47,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:556: done (1 left) 2014-07-01 16:52:47,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:557: done (0 left) 2014-07-01 16:52:47,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:558: snap_create 2014-07-01 16:52:48,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:2447: done (5 left) 2014-07-01 16:52:48,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: done (4 left) 2014-07-01 16:52:48,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: done (3 left) 2014-07-01 16:52:48,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2451: done (2 left) 2014-07-01 16:52:48,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: done (1 left) 2014-07-01 16:52:48,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2453: done (0 left) 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 68 current snap is 0 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: seq_num 1579 ranges {3145728=258048} 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: writing mira0435601-68 from 3145728 to 3403776 tid 1 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: seq_num 1580 ranges {6336512=778240,7114752=401408} 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:558: done (0 left) 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: write oid 287 current snap is 11 2014-07-01 16:52:48,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: seq_num 520 ranges {488939=774577,1867877=621062} 2014-07-01 16:52:48,522.522 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 16:52:48,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: writing mira0437250-287 from 488939 to 1263516 tid 1 2014-07-01 16:52:48,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: writing mira0437250-287 from 1867877 to 2488939 tid 2 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: snap_remove snap 7 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: finishing write tid 1 to mira0437250-287 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: finishing write tid 2 to mira0437250-287 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: finishing write tid 4 to mira0437250-287 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: finishing write tid 5 to mira0437250-287 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 113 (ObjNum 520 snap 11 seq_num 520) dirty exists 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: writing mira0435601-311 from 6336512 to 7114752 tid 1 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: writing mira0435601-311 from 7114752 to 7516160 tid 2 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 420 current snap is 0 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: seq_num 1581 ranges {4300800=802816,5103616=647168,5750784=430080,6180864=512000,6692864=16384} 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: writing mira0435601-420 from 4300800 to 5103616 tid 1 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: writing mira0435601-420 from 5103616 to 5750784 tid 2 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: writing mira0435601-420 from 5750784 to 6180864 tid 3 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: writing mira0435601-420 from 6180864 to 6692864 tid 4 2014-07-01 16:52:48,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: writing mira0435601-420 from 6692864 to 6709248 tid 5 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: seq_num 1582 ranges {2183168=430080,2613248=475136,3088384=327680} 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0435601-445 from 2183168 to 2613248 tid 1 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0435601-445 from 2613248 to 3088384 tid 2 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0435601-445 from 3088384 to 3416064 tid 3 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 295 current snap is 0 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: seq_num 1583 ranges {3555328=598016} 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: writing mira0435601-295 from 3555328 to 4153344 tid 1 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2459: read oid 219 snap -1 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: finishing write tid 1 to mira0435601-68 2014-07-01 16:52:48,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: finishing write tid 3 to mira0435601-68 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: finishing write tid 4 to mira0435601-68 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 145 (ObjNum 1579 snap 0 seq_num 1579) dirty exists 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: finishing write tid 1 to mira0435601-311 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: finishing write tid 2 to mira0435601-311 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: finishing write tid 4 to mira0435601-311 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: finishing write tid 5 to mira0435601-311 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 243 (ObjNum 1580 snap 0 seq_num 1580) dirty exists 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 1 to mira0435601-420 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 2 to mira0435601-420 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 3 to mira0435601-420 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: done (5 left) 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: done (4 left) 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 447 current snap is 0 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: seq_num 1584 ranges {0=622592,622592=487424,1110016=651264,1761280=65536} 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: writing mira0435601-447 from 0 to 622592 tid 1 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: writing mira0435601-447 from 622592 to 1110016 tid 2 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: writing mira0435601-447 from 1110016 to 1761280 tid 3 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: writing mira0435601-447 from 1761280 to 1826816 tid 4 2014-07-01 16:52:48,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 37 current snap is 0 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: seq_num 1585 ranges {1728512=651264} 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: writing mira0435601-37 from 1728512 to 2379776 tid 1 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 34 current snap is 0 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: seq_num 1586 ranges {2199552=770048,2969600=704512,3674112=98304} 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0435601-34 from 2199552 to 2969600 tid 1 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0435601-34 from 2969600 to 3674112 tid 2 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0435601-34 from 3674112 to 3772416 tid 3 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 430 current snap is 0 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: seq_num 1587 ranges {3620864=442368,4063232=688128,4751360=114688} 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0435601-430 from 3620864 to 4063232 tid 1 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0435601-430 from 4063232 to 4751360 tid 2 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0435601-430 from 4751360 to 4866048 tid 3 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: read oid 32 snap -1 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 4 to mira0435601-420 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 5 to mira0435601-420 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 480 current snap is 0 2014-07-01 16:52:48,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: seq_num 1588 ranges {0=69632} 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: writing mira0435601-480 from 0 to 69632 tid 1 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2466: delete oid 256 current snap is 0 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 7 to mira0435601-420 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: finishing write tid 8 to mira0435601-420 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 290 (ObjNum 1581 snap 0 seq_num 1581) dirty exists 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 1 to mira0435601-445 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 2 to mira0435601-445 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 3 to mira0435601-445 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 5 to mira0435601-445 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 6 to mira0435601-445 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 216 (ObjNum 1582 snap 0 seq_num 1582) dirty exists 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: finishing write tid 1 to mira0435601-295 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: finishing write tid 3 to mira0435601-295 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: finishing write tid 4 to mira0435601-295 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 266 (ObjNum 1583 snap 0 seq_num 1583) dirty exists 2014-07-01 16:52:48,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2459: expect (ObjNum 1408 snap 0 seq_num 1408) 2014-07-01 16:52:49,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:559: done (1 left) 2014-07-01 16:52:49,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:560: done (0 left) 2014-07-01 16:52:49,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: write oid 366 current snap is 11 2014-07-01 16:52:49,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: seq_num 521 ranges {585193=497806,1732410=452783} 2014-07-01 16:52:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: writing mira0437250-366 from 585193 to 1082999 tid 1 2014-07-01 16:52:49,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: writing mira0437250-366 from 1732410 to 2185193 tid 2 2014-07-01 16:52:49,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: snap_remove snap 8 2014-07-01 16:52:49,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: finishing write tid 1 to mira0437250-366 2014-07-01 16:52:49,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: finishing write tid 2 to mira0437250-366 2014-07-01 16:52:49,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: finishing write tid 4 to mira0437250-366 2014-07-01 16:52:49,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: finishing write tid 5 to mira0437250-366 2014-07-01 16:52:49,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 92 (ObjNum 521 snap 11 seq_num 521) dirty exists 2014-07-01 16:52:49,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 1 to mira0435601-447 2014-07-01 16:52:49,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 2 to mira0435601-447 2014-07-01 16:52:49,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 3 to mira0435601-447 2014-07-01 16:52:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 4 to mira0435601-447 2014-07-01 16:52:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 6 to mira0435601-447 2014-07-01 16:52:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: finishing write tid 7 to mira0435601-447 2014-07-01 16:52:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 269 (ObjNum 1584 snap 0 seq_num 1584) dirty exists 2014-07-01 16:52:49,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: finishing write tid 1 to mira0435601-37 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: done (10 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: done (9 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: done (8 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2459: done (7 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: done (6 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2466: done (5 left) 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 414 current snap is 0 2014-07-01 16:52:49,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: seq_num 1590 ranges {0=614400,614400=684032,1298432=598016,1896448=323584} 2014-07-01 16:52:50,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: writing mira0435601-414 from 0 to 614400 tid 1 2014-07-01 16:52:50,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: writing mira0435601-414 from 614400 to 1298432 tid 2 2014-07-01 16:52:50,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: writing mira0435601-414 from 1298432 to 1896448 tid 3 2014-07-01 16:52:50,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: writing mira0435601-414 from 1896448 to 2220032 tid 4 2014-07-01 16:52:50,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2468: read oid 141 snap -1 2014-07-01 16:52:50,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: finishing write tid 3 to mira0435601-37 2014-07-01 16:52:50,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: finishing write tid 4 to mira0435601-37 2014-07-01 16:52:50,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 119 (ObjNum 1585 snap 0 seq_num 1585) dirty exists 2014-07-01 16:52:50,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 1 to mira0435601-34 2014-07-01 16:52:50,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 2 to mira0435601-34 2014-07-01 16:52:50,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 3 to mira0435601-34 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 5 to mira0435601-34 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 6 to mira0435601-34 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 221 (ObjNum 1586 snap 0 seq_num 1586) dirty exists 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 1 to mira0435601-430 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 2 to mira0435601-430 2014-07-01 16:52:50,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 3 to mira0435601-430 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 5 to mira0435601-430 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 1 to mira0435601-480 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 6 to mira0435601-430 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 206 (ObjNum 1587 snap 0 seq_num 1587) dirty exists 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 3 to mira0435601-480 2014-07-01 16:52:50,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 4 to mira0435601-480 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 108 (ObjNum 1588 snap 0 seq_num 1588) dirty exists 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 1 to mira0435601-414 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 2 to mira0435601-414 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: done (6 left) 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: done (5 left) 2014-07-01 16:52:50,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: done (4 left) 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: done (3 left) 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: done (2 left) 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2469: read oid 50 snap -1 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2470: read oid 63 snap -1 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 199 current snap is 0 2014-07-01 16:52:50,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: seq_num 1591 ranges {589824=643072,1232896=401408} 2014-07-01 16:52:50,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:561: done (1 left) 2014-07-01 16:52:50,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:562: done (0 left) 2014-07-01 16:52:50,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:563: delete oid 410 current snap is 11 2014-07-01 16:52:50,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: writing mira0435601-199 from 589824 to 1232896 tid 1 2014-07-01 16:52:50,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: writing mira0435601-199 from 1232896 to 1634304 tid 2 2014-07-01 16:52:50,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 194 current snap is 0 2014-07-01 16:52:50,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: seq_num 1592 ranges {7245824=565248,7811072=446464,8257536=700416,8957952=57344} 2014-07-01 16:52:50,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:563: done (0 left) 2014-07-01 16:52:50,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:564: snap_create 2014-07-01 16:52:50,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: writing mira0435601-194 from 7245824 to 7811072 tid 1 2014-07-01 16:52:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: writing mira0435601-194 from 7811072 to 8257536 tid 2 2014-07-01 16:52:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: writing mira0435601-194 from 8257536 to 8957952 tid 3 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: writing mira0435601-194 from 8957952 to 9015296 tid 4 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2473: read oid 32 snap -1 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 3 to mira0435601-414 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 4 to mira0435601-414 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 6 to mira0435601-414 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: finishing write tid 7 to mira0435601-414 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 335 (ObjNum 1590 snap 0 seq_num 1590) dirty exists 2014-07-01 16:52:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: finishing write tid 1 to mira0435601-199 2014-07-01 16:52:50,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: finishing write tid 2 to mira0435601-199 2014-07-01 16:52:50,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: finishing write tid 4 to mira0435601-199 2014-07-01 16:52:50,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: finishing write tid 5 to mira0435601-199 2014-07-01 16:52:50,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 122 (ObjNum 1591 snap 0 seq_num 1591) dirty exists 2014-07-01 16:52:50,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:2469: expect (ObjNum 1390 snap 0 seq_num 1390) 2014-07-01 16:52:51,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:564: done (0 left) 2014-07-01 16:52:51,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: write oid 149 current snap is 12 2014-07-01 16:52:51,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: seq_num 523 ranges {190179=1} 2014-07-01 16:52:51,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: writing mira0437250-149 from 190179 to 190180 tid 1 2014-07-01 16:52:51,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: rollback oid 268 current snap is 12 2014-07-01 16:52:51,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 268 to 2 2014-07-01 16:52:51,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:567: delete oid 297 current snap is 12 2014-07-01 16:52:51,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: finishing write tid 1 to mira0437250-149 2014-07-01 16:52:51,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: finishing write tid 3 to mira0437250-149 2014-07-01 16:52:51,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: finishing write tid 4 to mira0437250-149 2014-07-01 16:52:51,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 141 (ObjNum 523 snap 12 seq_num 523) dirty exists 2014-07-01 16:52:51,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:565: done (2 left) 2014-07-01 16:52:51,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:567: done (1 left) 2014-07-01 16:52:51,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:568: rollback oid 264 current snap is 12 2014-07-01 16:52:51,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 264 to 11 2014-07-01 16:52:51,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:569: delete oid 81 current snap is 12 2014-07-01 16:52:51,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 94 (ObjNum 267 snap 0 seq_num 267) dirty exists 2014-07-01 16:52:51,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 143 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 16:52:51,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:566: done (2 left) 2014-07-01 16:52:51,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:568: done (1 left) 2014-07-01 16:52:51,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:569: done (0 left) 2014-07-01 16:52:51,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: write oid 472 current snap is 12 2014-07-01 16:52:51,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: seq_num 526 ranges {574045=677595,1374044=1} 2014-07-01 16:52:51,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: writing mira0437250-472 from 574045 to 1251640 tid 1 2014-07-01 16:52:51,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: writing mira0437250-472 from 1374044 to 1374045 tid 2 2014-07-01 16:52:51,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: write oid 416 current snap is 12 2014-07-01 16:52:51,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: seq_num 527 ranges {669275=400000} 2014-07-01 16:52:51,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: writing mira0437250-416 from 669275 to 1069275 tid 1 2014-07-01 16:52:51,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: write oid 189 current snap is 12 2014-07-01 16:52:51,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: seq_num 528 ranges {730457=772628,2121222=209235} 2014-07-01 16:52:51,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0437250-189 from 730457 to 1503085 tid 1 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: writing mira0437250-189 from 2121222 to 2330457 tid 2 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:573: delete oid 297 current snap is 12 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 1 to mira0437250-472 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 2 to mira0437250-472 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 4 to mira0437250-472 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: finishing write tid 5 to mira0437250-472 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 99 (ObjNum 526 snap 12 seq_num 526) dirty exists 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: finishing write tid 1 to mira0437250-416 2014-07-01 16:52:51,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: finishing write tid 3 to mira0437250-416 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: finishing write tid 4 to mira0437250-416 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 91 (ObjNum 527 snap 12 seq_num 527) dirty exists 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 1 to mira0437250-189 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 2 to mira0437250-189 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:570: done (3 left) 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:571: done (2 left) 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:573: done (1 left) 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:574: snap_create 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 4 to mira0437250-189 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: finishing write tid 5 to mira0437250-189 2014-07-01 16:52:51,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 148 (ObjNum 528 snap 12 seq_num 528) dirty exists 2014-07-01 16:52:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:572: done (1 left) 2014-07-01 16:52:52,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:574: done (0 left) 2014-07-01 16:52:52,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:575: snap_remove snap 0 2014-07-01 16:52:53,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:575: done (0 left) 2014-07-01 16:52:53,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: write oid 493 current snap is 13 2014-07-01 16:52:53,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: seq_num 530 ranges {770516=752333,2197038=526630,2770515=1} 2014-07-01 16:52:53,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: writing mira0437250-493 from 770516 to 1522849 tid 1 2014-07-01 16:52:53,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: writing mira0437250-493 from 2197038 to 2723668 tid 2 2014-07-01 16:52:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: writing mira0437250-493 from 2770515 to 2770516 tid 3 2014-07-01 16:52:53,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: read oid 98 snap -1 2014-07-01 16:52:53,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:578: snap_create 2014-07-01 16:52:53,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 16:52:53,521.521 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:53,521.521 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:53,521.521 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 16:52:53,521.521 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 16:52:54,110.110 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 16:52:54,139.139 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 16:52:54,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: finishing write tid 1 to mira0437250-493 2014-07-01 16:52:54,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: finishing write tid 2 to mira0437250-493 2014-07-01 16:52:54,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: finishing write tid 3 to mira0437250-493 2014-07-01 16:52:54,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: finishing write tid 5 to mira0437250-493 2014-07-01 16:52:54,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: finishing write tid 6 to mira0437250-493 2014-07-01 16:52:54,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 155 (ObjNum 530 snap 13 seq_num 530) dirty exists 2014-07-01 16:52:54,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:576: done (2 left) 2014-07-01 16:52:54,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:577: done (1 left) 2014-07-01 16:52:54,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:578: done (0 left) 2014-07-01 16:52:54,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: write oid 421 current snap is 14 2014-07-01 16:52:54,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: seq_num 531 ranges {715346=475561,1911170=748249,3414654=100692} 2014-07-01 16:52:54,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: writing mira0437250-421 from 715346 to 1190907 tid 1 2014-07-01 16:52:54,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: writing mira0437250-421 from 1911170 to 2659419 tid 2 2014-07-01 16:52:54,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: writing mira0437250-421 from 3414654 to 3515346 tid 3 2014-07-01 16:52:54,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:580: rollback oid 16 current snap is 14 2014-07-01 16:52:54,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 16 to 11 2014-07-01 16:52:54,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: delete oid 355 current snap is 14 2014-07-01 16:52:54,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 97 (ObjNum 15 snap 0 seq_num 15) dirty exists 2014-07-01 16:52:55,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 1 to mira0435601-194 2014-07-01 16:52:55,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 2 to mira0435601-194 2014-07-01 16:52:55,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 3 to mira0435601-194 2014-07-01 16:52:55,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 4 to mira0435601-194 2014-07-01 16:52:55,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 6 to mira0435601-194 2014-07-01 16:52:55,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 7 to mira0435601-194 2014-07-01 16:52:55,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 194 v 273 (ObjNum 1592 snap 0 seq_num 1592) dirty exists 2014-07-01 16:52:55,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: done (6 left) 2014-07-01 16:52:55,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2468: done (5 left) 2014-07-01 16:52:55,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2469: done (4 left) 2014-07-01 16:52:55,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2470: done (3 left) 2014-07-01 16:52:55,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: done (2 left) 2014-07-01 16:52:55,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: done (1 left) 2014-07-01 16:52:55,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2473: done (0 left) 2014-07-01 16:52:55,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:52:55,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: seq_num 1593 ranges {4145152=434176,4579328=434176,5013504=524288,5537792=245760} 2014-07-01 16:52:55,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: writing mira0435601-27 from 4145152 to 4579328 tid 1 2014-07-01 16:52:55,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: writing mira0435601-27 from 4579328 to 5013504 tid 2 2014-07-01 16:52:55,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: writing mira0435601-27 from 5013504 to 5537792 tid 3 2014-07-01 16:52:55,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: writing mira0435601-27 from 5537792 to 5783552 tid 4 2014-07-01 16:52:55,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2475: read oid 193 snap -1 2014-07-01 16:52:55,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2476: read oid 500 snap -1 2014-07-01 16:52:55,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 105 current snap is 0 2014-07-01 16:52:55,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: seq_num 1594 ranges {2183168=462848} 2014-07-01 16:52:55,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: writing mira0435601-105 from 2183168 to 2646016 tid 1 2014-07-01 16:52:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 330 current snap is 0 2014-07-01 16:52:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: seq_num 1595 ranges {1757184=335872} 2014-07-01 16:52:55,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: writing mira0435601-330 from 1757184 to 2093056 tid 1 2014-07-01 16:52:55,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 77 current snap is 0 2014-07-01 16:52:55,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: seq_num 1596 ranges {2019328=655360,2674688=401408} 2014-07-01 16:52:55,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: writing mira0435601-77 from 2019328 to 2674688 tid 1 2014-07-01 16:52:55,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: writing mira0435601-77 from 2674688 to 3076096 tid 2 2014-07-01 16:52:55,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: read oid 481 snap -1 2014-07-01 16:52:55,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: read oid 306 snap -1 2014-07-01 16:52:55,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2475: expect (ObjNum 1347 snap 0 seq_num 1347) 2014-07-01 16:52:57,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: finishing write tid 1 to mira0437250-421 2014-07-01 16:52:57,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: finishing write tid 2 to mira0437250-421 2014-07-01 16:52:57,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: finishing write tid 3 to mira0437250-421 2014-07-01 16:52:57,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: finishing write tid 5 to mira0437250-421 2014-07-01 16:52:57,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:580: done (2 left) 2014-07-01 16:52:57,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:581: done (1 left) 2014-07-01 16:52:57,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: delete oid 237 current snap is 14 2014-07-01 16:52:57,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: finishing write tid 6 to mira0437250-421 2014-07-01 16:52:57,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 164 (ObjNum 531 snap 14 seq_num 531) dirty exists 2014-07-01 16:52:57,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:579: done (1 left) 2014-07-01 16:52:57,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:582: done (0 left) 2014-07-01 16:52:57,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: write oid 109 current snap is 14 2014-07-01 16:52:57,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: seq_num 534 ranges {415563=428094,1215562=1} 2014-07-01 16:52:57,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: writing mira0437250-109 from 415563 to 843657 tid 1 2014-07-01 16:52:57,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: writing mira0437250-109 from 1215562 to 1215563 tid 2 2014-07-01 16:52:57,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:584: snap_remove snap 2 2014-07-01 16:52:57,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:584: done (1 left) 2014-07-01 16:52:57,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: rollback oid 55 current snap is 14 2014-07-01 16:52:57,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 55 to 11 2014-07-01 16:52:57,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: write oid 17 current snap is 14 2014-07-01 16:52:57,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: seq_num 535 ranges {759369=799898,1959368=1} 2014-07-01 16:52:57,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: writing mira0437250-17 from 759369 to 1559267 tid 1 2014-07-01 16:52:57,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: writing mira0437250-17 from 1959368 to 1959369 tid 2 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: write oid 163 current snap is 14 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: seq_num 536 ranges {455623=771703,1655622=1} 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: writing mira0437250-163 from 455623 to 1227326 tid 1 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: writing mira0437250-163 from 1655622 to 1655623 tid 2 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: read oid 181 snap -1 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:589: snap_remove snap 13 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 1 to mira0437250-17 2014-07-01 16:52:57,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 2 to mira0437250-17 2014-07-01 16:52:57,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 4 to mira0437250-17 2014-07-01 16:52:57,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: expect (ObjNum 180 snap 0 seq_num 180) 2014-07-01 16:52:57,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 1 to mira0437250-163 2014-07-01 16:52:57,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 2 to mira0437250-163 2014-07-01 16:52:57,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 4 to mira0437250-163 2014-07-01 16:52:57,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: finishing write tid 5 to mira0437250-17 2014-07-01 16:52:57,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 17 v 180 (ObjNum 535 snap 14 seq_num 535) dirty exists 2014-07-01 16:52:57,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: finishing write tid 5 to mira0437250-163 2014-07-01 16:52:57,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 184 (ObjNum 536 snap 14 seq_num 536) dirty exists 2014-07-01 16:52:57,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: expect (ObjNum 1434 snap 0 seq_num 1434) 2014-07-01 16:52:58,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:586: done (5 left) 2014-07-01 16:52:58,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:587: done (4 left) 2014-07-01 16:52:58,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:588: done (3 left) 2014-07-01 16:52:58,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:589: done (2 left) 2014-07-01 16:52:58,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:590: snap_remove snap 12 2014-07-01 16:52:59,140.140 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:52:59,140.140 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:52:59,140.140 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 16:52:59,141.141 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:52:59,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: expect (ObjNum 1187 snap 0 seq_num 1187) 2014-07-01 16:52:59,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:590: done (2 left) 2014-07-01 16:52:59,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: write oid 404 current snap is 14 2014-07-01 16:52:59,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: seq_num 537 ranges {515780=495955,1653627=554892,2915779=1} 2014-07-01 16:52:59,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: writing mira0437250-404 from 515780 to 1011735 tid 1 2014-07-01 16:52:59,914.914 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:52:59,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: writing mira0437250-404 from 1653627 to 2208519 tid 2 2014-07-01 16:52:59,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: writing mira0437250-404 from 2915779 to 2915780 tid 3 2014-07-01 16:52:59,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: write oid 364 current snap is 14 2014-07-01 16:52:59,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: seq_num 538 ranges {460610=467759,1616335=625088,2982464=411952,3660609=1} 2014-07-01 16:52:59,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: writing mira0437250-364 from 460610 to 928369 tid 1 2014-07-01 16:52:59,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: writing mira0437250-364 from 1616335 to 2241423 tid 2 2014-07-01 16:52:59,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: writing mira0437250-364 from 2982464 to 3394416 tid 3 2014-07-01 16:53:00,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: writing mira0437250-364 from 3660609 to 3660610 tid 4 2014-07-01 16:53:00,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: write oid 428 current snap is 14 2014-07-01 16:53:00,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: seq_num 539 ranges {556864=474636,1614116=446707,2818763=538101} 2014-07-01 16:53:00,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: writing mira0437250-428 from 556864 to 1031500 tid 1 2014-07-01 16:53:00,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: writing mira0437250-428 from 1614116 to 2060823 tid 2 2014-07-01 16:53:00,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: writing mira0437250-428 from 2818763 to 3356864 tid 3 2014-07-01 16:53:00,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: snap_create 2014-07-01 16:53:00,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: finishing write tid 1 to mira0437250-404 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: finishing write tid 2 to mira0437250-404 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: finishing write tid 3 to mira0437250-404 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: finishing write tid 5 to mira0437250-404 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: finishing write tid 6 to mira0437250-404 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 100 (ObjNum 537 snap 14 seq_num 537) dirty exists 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 1 to mira0437250-364 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 2 to mira0437250-364 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 3 to mira0437250-364 2014-07-01 16:53:00,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 4 to mira0437250-364 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 6 to mira0437250-364 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: finishing write tid 7 to mira0437250-364 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 364 v 198 (ObjNum 538 snap 14 seq_num 538) dirty exists 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: finishing write tid 1 to mira0437250-428 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: finishing write tid 1 to mira0437250-109 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: finishing write tid 2 to mira0437250-109 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: finishing write tid 4 to mira0437250-109 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: finishing write tid 5 to mira0437250-109 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 168 (ObjNum 534 snap 14 seq_num 534) dirty exists 2014-07-01 16:53:00,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: finishing write tid 2 to mira0437250-428 2014-07-01 16:53:00,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: finishing write tid 3 to mira0437250-428 2014-07-01 16:53:00,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: finishing write tid 5 to mira0437250-428 2014-07-01 16:53:00,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: finishing write tid 6 to mira0437250-428 2014-07-01 16:53:00,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 112 (ObjNum 539 snap 14 seq_num 539) dirty exists 2014-07-01 16:53:00,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 170 (ObjNum 54 snap 0 seq_num 54) dirty exists 2014-07-01 16:53:01,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:583: done (5 left) 2014-07-01 16:53:01,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:585: done (4 left) 2014-07-01 16:53:01,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:591: done (3 left) 2014-07-01 16:53:01,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:592: done (2 left) 2014-07-01 16:53:01,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:593: done (1 left) 2014-07-01 16:53:01,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:594: done (0 left) 2014-07-01 16:53:01,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: write oid 301 current snap is 15 2014-07-01 16:53:01,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: seq_num 540 ranges {99645=1} 2014-07-01 16:53:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: writing mira0437250-301 from 99645 to 99646 tid 1 2014-07-01 16:53:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: read oid 371 snap -1 2014-07-01 16:53:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: write oid 307 current snap is 15 2014-07-01 16:53:01,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: seq_num 541 ranges {312250=1} 2014-07-01 16:53:01,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: writing mira0437250-307 from 312250 to 312251 tid 1 2014-07-01 16:53:01,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: read oid 147 snap 6 2014-07-01 16:53:01,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: snap_remove snap 10 2014-07-01 16:53:01,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: finishing write tid 1 to mira0437250-307 2014-07-01 16:53:01,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: finishing write tid 1 to mira0435601-77 2014-07-01 16:53:01,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: finishing write tid 2 to mira0435601-77 2014-07-01 16:53:01,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: finishing write tid 4 to mira0435601-77 2014-07-01 16:53:01,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: finishing write tid 1 to mira0435601-330 2014-07-01 16:53:01,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: finishing write tid 5 to mira0435601-77 2014-07-01 16:53:01,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 210 (ObjNum 1596 snap 0 seq_num 1596) dirty exists 2014-07-01 16:53:01,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: finishing write tid 3 to mira0435601-330 2014-07-01 16:53:01,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: finishing write tid 4 to mira0435601-330 2014-07-01 16:53:01,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 250 (ObjNum 1595 snap 0 seq_num 1595) dirty exists 2014-07-01 16:53:01,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: finishing write tid 1 to mira0435601-105 2014-07-01 16:53:01,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: finishing write tid 3 to mira0435601-105 2014-07-01 16:53:01,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: finishing write tid 4 to mira0435601-105 2014-07-01 16:53:01,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 137 (ObjNum 1594 snap 0 seq_num 1594) dirty exists 2014-07-01 16:53:01,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 1 to mira0435601-27 2014-07-01 16:53:01,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 2 to mira0435601-27 2014-07-01 16:53:01,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 3 to mira0435601-27 2014-07-01 16:53:01,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 4 to mira0435601-27 2014-07-01 16:53:01,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 6 to mira0435601-27 2014-07-01 16:53:01,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: finishing write tid 7 to mira0435601-27 2014-07-01 16:53:01,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 221 (ObjNum 1593 snap 0 seq_num 1593) dirty exists 2014-07-01 16:53:01,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: done (7 left) 2014-07-01 16:53:01,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2475: done (6 left) 2014-07-01 16:53:01,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2476: done (5 left) 2014-07-01 16:53:01,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: done (4 left) 2014-07-01 16:53:01,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: done (3 left) 2014-07-01 16:53:01,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: done (2 left) 2014-07-01 16:53:01,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: done (1 left) 2014-07-01 16:53:01,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: done (0 left) 2014-07-01 16:53:01,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2482: read oid 152 snap -1 2014-07-01 16:53:01,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2483: read oid 410 snap -1 2014-07-01 16:53:01,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 162 current snap is 0 2014-07-01 16:53:01,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: seq_num 1597 ranges {2215936=684032,2899968=659456,3559424=417792,3977216=528384} 2014-07-01 16:53:01,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: writing mira0435601-162 from 2215936 to 2899968 tid 1 2014-07-01 16:53:01,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: writing mira0435601-162 from 2899968 to 3559424 tid 2 2014-07-01 16:53:01,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: writing mira0435601-162 from 3559424 to 3977216 tid 3 2014-07-01 16:53:01,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: writing mira0435601-162 from 3977216 to 4505600 tid 4 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: read oid 193 snap -1 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2482: done (3 left) 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: read oid 407 snap -1 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2487: read oid 139 snap -1 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 241 current snap is 0 2014-07-01 16:53:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: seq_num 1598 ranges {5976064=552960,6529024=487424,7016448=647168,7663616=471040} 2014-07-01 16:53:01,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: writing mira0435601-241 from 5976064 to 6529024 tid 1 2014-07-01 16:53:01,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: finishing write tid 3 to mira0437250-307 2014-07-01 16:53:01,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: finishing write tid 4 to mira0437250-307 2014-07-01 16:53:01,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 201 (ObjNum 541 snap 15 seq_num 541) dirty exists 2014-07-01 16:53:01,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: writing mira0435601-241 from 6529024 to 7016448 tid 2 2014-07-01 16:53:01,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: writing mira0435601-241 from 7016448 to 7663616 tid 3 2014-07-01 16:53:01,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: writing mira0435601-241 from 7663616 to 8134656 tid 4 2014-07-01 16:53:01,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2489: read oid 246 snap -1 2014-07-01 16:53:01,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: expect (ObjNum 1531 snap 0 seq_num 1531) 2014-07-01 16:53:02,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:597: done (4 left) 2014-07-01 16:53:02,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:599: done (3 left) 2014-07-01 16:53:02,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: read oid 471 snap -1 2014-07-01 16:53:02,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: read oid 335 snap -1 2014-07-01 16:53:02,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: write oid 25 current snap is 15 2014-07-01 16:53:02,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: seq_num 542 ranges {657081=400000} 2014-07-01 16:53:02,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: writing mira0437250-25 from 657081 to 1057081 tid 1 2014-07-01 16:53:02,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:603: rollback oid 296 current snap is 15 2014-07-01 16:53:02,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 296 to 9 2014-07-01 16:53:02,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: write oid 105 current snap is 15 2014-07-01 16:53:02,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: seq_num 543 ranges {600887=549373,1598452=202435} 2014-07-01 16:53:02,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0437250-105 from 600887 to 1150260 tid 1 2014-07-01 16:53:02,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: writing mira0437250-105 from 1598452 to 1800887 tid 2 2014-07-01 16:53:02,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: read oid 276 snap -1 2014-07-01 16:53:02,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: expect (ObjNum 470 snap 0 seq_num 470) 2014-07-01 16:53:02,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 1 to mira0435601-241 2014-07-01 16:53:02,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 2 to mira0435601-241 2014-07-01 16:53:02,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: expect (ObjNum 1347 snap 0 seq_num 1347) 2014-07-01 16:53:02,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: expect (ObjNum 275 snap 0 seq_num 275) 2014-07-01 16:53:03,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 115 (ObjNum 295 snap 0 seq_num 295) dirty exists 2014-07-01 16:53:03,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: expect (ObjNum 334 snap 0 seq_num 334) 2014-07-01 16:53:05,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 3 to mira0435601-241 2014-07-01 16:53:05,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 4 to mira0435601-241 2014-07-01 16:53:05,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 6 to mira0435601-241 2014-07-01 16:53:05,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2489: expect (ObjNum 857 snap 0 seq_num 857) 2014-07-01 16:53:05,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 1 to mira0437250-105 2014-07-01 16:53:05,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 2 to mira0437250-105 2014-07-01 16:53:05,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 4 to mira0437250-105 2014-07-01 16:53:05,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: finishing write tid 5 to mira0437250-105 2014-07-01 16:53:05,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 106 (ObjNum 543 snap 15 seq_num 543) dirty exists 2014-07-01 16:53:05,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 1 to mira0437250-25 2014-07-01 16:53:05,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 3 to mira0437250-25 2014-07-01 16:53:05,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:600: done (8 left) 2014-07-01 16:53:05,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:601: done (7 left) 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:603: done (6 left) 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:604: done (5 left) 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:605: done (4 left) 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: write oid 213 current snap is 15 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: seq_num 544 ranges {697141=522202,1497140=1} 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: writing mira0437250-213 from 697141 to 1219343 tid 1 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: writing mira0437250-213 from 1497140 to 1497141 tid 2 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: write oid 208 current snap is 15 2014-07-01 16:53:05,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: seq_num 545 ranges {640946=645430,1790620=450326} 2014-07-01 16:53:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: writing mira0437250-208 from 640946 to 1286376 tid 1 2014-07-01 16:53:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: writing mira0437250-208 from 1790620 to 2240946 tid 2 2014-07-01 16:53:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: snap_remove snap 14 2014-07-01 16:53:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: finishing write tid 4 to mira0437250-25 2014-07-01 16:53:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 104 (ObjNum 542 snap 15 seq_num 542) dirty exists 2014-07-01 16:53:05,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: expect (ObjNum 370 snap 0 seq_num 370) 2014-07-01 16:53:05,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 1 to mira0437250-208 2014-07-01 16:53:05,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 2 to mira0437250-208 2014-07-01 16:53:05,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 4 to mira0437250-208 2014-07-01 16:53:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: finishing write tid 5 to mira0437250-208 2014-07-01 16:53:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 145 (ObjNum 545 snap 15 seq_num 545) dirty exists 2014-07-01 16:53:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: finishing write tid 1 to mira0437250-213 2014-07-01 16:53:05,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: finishing write tid 2 to mira0437250-213 2014-07-01 16:53:05,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: finishing write tid 4 to mira0437250-213 2014-07-01 16:53:05,527.527 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 46' 2014-07-01 16:53:05,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: finishing write tid 7 to mira0435601-241 2014-07-01 16:53:05,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 219 (ObjNum 1598 snap 0 seq_num 1598) dirty exists 2014-07-01 16:53:05,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 1 to mira0435601-162 2014-07-01 16:53:05,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 2 to mira0435601-162 2014-07-01 16:53:05,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 3 to mira0435601-162 2014-07-01 16:53:05,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 4 to mira0435601-162 2014-07-01 16:53:05,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: done (6 left) 2014-07-01 16:53:05,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: done (5 left) 2014-07-01 16:53:05,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2487: done (4 left) 2014-07-01 16:53:05,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: done (3 left) 2014-07-01 16:53:05,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2489: done (2 left) 2014-07-01 16:53:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2490: read oid 433 snap -1 2014-07-01 16:53:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 153 current snap is 0 2014-07-01 16:53:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: seq_num 1599 ranges {2871296=425984,3297280=475136,3772416=765952,4538368=364544} 2014-07-01 16:53:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: writing mira0435601-153 from 2871296 to 3297280 tid 1 2014-07-01 16:53:05,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: writing mira0435601-153 from 3297280 to 3772416 tid 2 2014-07-01 16:53:05,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: writing mira0435601-153 from 3772416 to 4538368 tid 3 2014-07-01 16:53:05,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: writing mira0435601-153 from 4538368 to 4902912 tid 4 2014-07-01 16:53:05,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:2492: read oid 114 snap -1 2014-07-01 16:53:05,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 6 to mira0435601-162 2014-07-01 16:53:05,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 7 to mira0435601-162 2014-07-01 16:53:05,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 238 (ObjNum 1597 snap 0 seq_num 1597) dirty exists 2014-07-01 16:53:05,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:2483: expect (ObjNum 1361 snap 0 seq_num 1361) 2014-07-01 16:53:06,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: finishing write tid 5 to mira0437250-213 2014-07-01 16:53:06,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 122 (ObjNum 544 snap 15 seq_num 544) dirty exists 2014-07-01 16:53:06,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:596: done (6 left) 2014-07-01 16:53:06,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:602: done (5 left) 2014-07-01 16:53:06,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:606: done (4 left) 2014-07-01 16:53:06,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:607: done (3 left) 2014-07-01 16:53:06,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:608: done (2 left) 2014-07-01 16:53:06,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: delete oid 447 current snap is 15 2014-07-01 16:53:06,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:2483: done (4 left) 2014-07-01 16:53:06,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: done (3 left) 2014-07-01 16:53:06,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2490: done (2 left) 2014-07-01 16:53:06,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: read oid 31 snap -1 2014-07-01 16:53:06,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2494: delete oid 498 current snap is 0 2014-07-01 16:53:06,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: expect (ObjNum 1021 snap 0 seq_num 1021) 2014-07-01 16:53:07,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:609: done (2 left) 2014-07-01 16:53:07,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: snap_create 2014-07-01 16:53:07,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: expect (ObjNum 146 snap 0 seq_num 146) 2014-07-01 16:53:07,850.850 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 46 2014-07-01 16:53:08,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: done (3 left) 2014-07-01 16:53:08,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2494: done (2 left) 2014-07-01 16:53:08,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2495: delete oid 426 current snap is 0 2014-07-01 16:53:08,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 1 to mira0437250-301 2014-07-01 16:53:08,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 3 to mira0437250-301 2014-07-01 16:53:08,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: finishing write tid 4 to mira0437250-301 2014-07-01 16:53:08,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 129 (ObjNum 540 snap 15 seq_num 540) dirty exists 2014-07-01 16:53:08,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:595: done (2 left) 2014-07-01 16:53:08,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:598: done (1 left) 2014-07-01 16:53:08,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:610: done (0 left) 2014-07-01 16:53:08,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: snap_remove snap 6 2014-07-01 16:53:08,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:611: done (0 left) 2014-07-01 16:53:08,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: snap_create 2014-07-01 16:53:09,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2495: done (2 left) 2014-07-01 16:53:09,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:2496: read oid 160 snap -1 2014-07-01 16:53:09,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: read oid 82 snap -1 2014-07-01 16:53:09,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2498: read oid 182 snap -1 2014-07-01 16:53:09,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2499: read oid 285 snap -1 2014-07-01 16:53:09,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2500: read oid 250 snap -1 2014-07-01 16:53:09,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2501: delete oid 379 current snap is 0 2014-07-01 16:53:09,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2501: done (7 left) 2014-07-01 16:53:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 106 current snap is 0 2014-07-01 16:53:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: seq_num 1603 ranges {1339392=274432} 2014-07-01 16:53:09,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: writing mira0435601-106 from 1339392 to 1613824 tid 1 2014-07-01 16:53:09,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 219 current snap is 0 2014-07-01 16:53:09,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: seq_num 1604 ranges {2789376=143360} 2014-07-01 16:53:09,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: writing mira0435601-219 from 2789376 to 2932736 tid 1 2014-07-01 16:53:09,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 197 current snap is 0 2014-07-01 16:53:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: seq_num 1605 ranges {0=622592,622592=466944,1089536=712704,1802240=425984} 2014-07-01 16:53:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: writing mira0435601-197 from 0 to 622592 tid 1 2014-07-01 16:53:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: writing mira0435601-197 from 622592 to 1089536 tid 2 2014-07-01 16:53:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: writing mira0435601-197 from 1089536 to 1802240 tid 3 2014-07-01 16:53:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: writing mira0435601-197 from 1802240 to 2228224 tid 4 2014-07-01 16:53:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 161 current snap is 0 2014-07-01 16:53:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: seq_num 1606 ranges {1728512=741376} 2014-07-01 16:53:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: writing mira0435601-161 from 1728512 to 2469888 tid 1 2014-07-01 16:53:09,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: read oid 362 snap -1 2014-07-01 16:53:09,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:2499: expect (ObjNum 1475 snap 0 seq_num 1475) 2014-07-01 16:53:09,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 1 to mira0435601-106 2014-07-01 16:53:09,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 3 to mira0435601-106 2014-07-01 16:53:09,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:2499: done (11 left) 2014-07-01 16:53:09,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:53:09,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: seq_num 1607 ranges {1277952=765952,2043904=679936,2723840=524288} 2014-07-01 16:53:09,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:612: done (0 left) 2014-07-01 16:53:09,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: read oid 481 snap -1 2014-07-01 16:53:09,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: read oid 137 snap 9 2014-07-01 16:53:09,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: snap_create 2014-07-01 16:53:09,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: writing mira0435601-284 from 1277952 to 2043904 tid 1 2014-07-01 16:53:09,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: writing mira0435601-284 from 2043904 to 2723840 tid 2 2014-07-01 16:53:09,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: writing mira0435601-284 from 2723840 to 3248128 tid 3 2014-07-01 16:53:09,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2508: read oid 59 snap -1 2014-07-01 16:53:09,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2509: read oid 264 snap -1 2014-07-01 16:53:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: read oid 91 snap -1 2014-07-01 16:53:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2511: read oid 323 snap -1 2014-07-01 16:53:09,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:53:09,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2508: done (15 left) 2014-07-01 16:53:09,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2512: read oid 49 snap -1 2014-07-01 16:53:09,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:53:09,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 4 to mira0435601-106 2014-07-01 16:53:09,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 240 (ObjNum 1603 snap 0 seq_num 1603) dirty exists 2014-07-01 16:53:09,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: done (15 left) 2014-07-01 16:53:09,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 231 current snap is 0 2014-07-01 16:53:09,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: seq_num 1608 ranges {1781760=794624} 2014-07-01 16:53:09,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: writing mira0435601-231 from 1781760 to 2576384 tid 1 2014-07-01 16:53:09,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:53:09,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 1 to mira0435601-197 2014-07-01 16:53:09,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2511: done (15 left) 2014-07-01 16:53:09,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 16:53:10,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: seq_num 1609 ranges {7745536=512000,8257536=446464,8704000=606208,9310208=151552} 2014-07-01 16:53:10,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: expect (ObjNum 480 snap 0 seq_num 480) 2014-07-01 16:53:10,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: writing mira0435601-172 from 7745536 to 8257536 tid 1 2014-07-01 16:53:10,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: writing mira0435601-172 from 8257536 to 8704000 tid 2 2014-07-01 16:53:10,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: writing mira0435601-172 from 8704000 to 9310208 tid 3 2014-07-01 16:53:10,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: writing mira0435601-172 from 9310208 to 9461760 tid 4 2014-07-01 16:53:10,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 16:53:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2496: expect (ObjNum 814 snap 0 seq_num 814) 2014-07-01 16:53:10,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:613: done (2 left) 2014-07-01 16:53:10,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:615: done (1 left) 2014-07-01 16:53:10,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:616: snap_create 2014-07-01 16:53:11,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 2 to mira0435601-197 2014-07-01 16:53:11,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 3 to mira0435601-197 2014-07-01 16:53:11,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 4 to mira0435601-197 2014-07-01 16:53:11,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 6 to mira0435601-197 2014-07-01 16:53:11,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:2509: expect (ObjNum 677 snap 0 seq_num 677) 2014-07-01 16:53:11,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:616: done (1 left) 2014-07-01 16:53:11,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: write oid 394 current snap is 19 2014-07-01 16:53:11,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: seq_num 547 ranges {798382=472687,1983813=794764,3198381=1} 2014-07-01 16:53:11,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: writing mira0437250-394 from 798382 to 1271069 tid 1 2014-07-01 16:53:11,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: writing mira0437250-394 from 1983813 to 2778577 tid 2 2014-07-01 16:53:11,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: writing mira0437250-394 from 3198381 to 3198382 tid 3 2014-07-01 16:53:11,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:618: snap_create 2014-07-01 16:53:11,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2498: expect (ObjNum 1555 snap 0 seq_num 1555) 2014-07-01 16:53:12,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: finishing write tid 1 to mira0435601-161 2014-07-01 16:53:12,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: finishing write tid 3 to mira0435601-161 2014-07-01 16:53:12,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: expect (ObjNum 1352 snap 0 seq_num 1352) 2014-07-01 16:53:12,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 1 to mira0437250-394 2014-07-01 16:53:12,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 2 to mira0437250-394 2014-07-01 16:53:12,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 3 to mira0437250-394 2014-07-01 16:53:12,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 5 to mira0437250-394 2014-07-01 16:53:12,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: finishing write tid 6 to mira0437250-394 2014-07-01 16:53:12,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 228 (ObjNum 547 snap 19 seq_num 547) dirty exists 2014-07-01 16:53:12,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:617: done (2 left) 2014-07-01 16:53:12,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:618: done (1 left) 2014-07-01 16:53:12,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:619: read oid 200 snap -1 2014-07-01 16:53:12,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: snap_create 2014-07-01 16:53:12,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:619: expect (ObjNum 199 snap 0 seq_num 199) 2014-07-01 16:53:12,848.848 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:12,849.849 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:12,849.849 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2014-07-01 16:53:12,849.849 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0' 2014-07-01 16:53:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: expect (ObjNum 136 snap 0 seq_num 136) 2014-07-01 16:53:13,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:614: done (2 left) 2014-07-01 16:53:13,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:619: done (1 left) 2014-07-01 16:53:13,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:620: done (0 left) 2014-07-01 16:53:13,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: write oid 49 current snap is 21 2014-07-01 16:53:13,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: seq_num 548 ranges {589740=400000} 2014-07-01 16:53:13,790.790 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0 (802) 2014-07-01 16:53:13,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: writing mira0437250-49 from 589740 to 989740 tid 1 2014-07-01 16:53:13,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: snap_create 2014-07-01 16:53:14,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:622: done (1 left) 2014-07-01 16:53:14,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:623: snap_create 2014-07-01 16:53:14,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 1 to mira0435601-231 2014-07-01 16:53:14,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: finishing write tid 7 to mira0435601-197 2014-07-01 16:53:14,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 197 v 260 (ObjNum 1605 snap 0 seq_num 1605) dirty exists 2014-07-01 16:53:14,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 3 to mira0435601-231 2014-07-01 16:53:14,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 1 to mira0435601-172 2014-07-01 16:53:14,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 2 to mira0435601-172 2014-07-01 16:53:14,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 3 to mira0435601-172 2014-07-01 16:53:14,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 4 to mira0435601-172 2014-07-01 16:53:14,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 6 to mira0435601-172 2014-07-01 16:53:14,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2496: done (15 left) 2014-07-01 16:53:14,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2498: done (14 left) 2014-07-01 16:53:14,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2500: done (13 left) 2014-07-01 16:53:14,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: done (12 left) 2014-07-01 16:53:14,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: done (11 left) 2014-07-01 16:53:14,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2509: done (10 left) 2014-07-01 16:53:14,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: read oid 84 snap -1 2014-07-01 16:53:14,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: finishing write tid 7 to mira0435601-172 2014-07-01 16:53:14,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 225 (ObjNum 1609 snap 0 seq_num 1609) dirty exists 2014-07-01 16:53:14,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:2492: expect (ObjNum 1383 snap 0 seq_num 1383) 2014-07-01 16:53:15,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: finishing write tid 1 to mira0437250-49 2014-07-01 16:53:15,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: finishing write tid 3 to mira0437250-49 2014-07-01 16:53:15,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:623: done (1 left) 2014-07-01 16:53:15,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: write oid 337 current snap is 23 2014-07-01 16:53:15,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: seq_num 549 ranges {685992=1} 2014-07-01 16:53:15,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: writing mira0437250-337 from 685992 to 685993 tid 1 2014-07-01 16:53:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: rollback oid 82 current snap is 23 2014-07-01 16:53:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 82 to 9 2014-07-01 16:53:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: write oid 452 current snap is 23 2014-07-01 16:53:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: seq_num 550 ranges {497575=400000} 2014-07-01 16:53:15,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: writing mira0437250-452 from 497575 to 897575 tid 1 2014-07-01 16:53:15,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: read oid 420 snap -1 2014-07-01 16:53:15,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:628: snap_create 2014-07-01 16:53:15,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 0 (ObjNum 16661840 snap 0 seq_num 15783728) dirty dne 2014-07-01 16:53:15,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: expect (ObjNum 419 snap 0 seq_num 419) 2014-07-01 16:53:16,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: finishing write tid 1 to mira0437250-337 2014-07-01 16:53:16,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: finishing write tid 3 to mira0437250-337 2014-07-01 16:53:16,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 4 to mira0435601-231 2014-07-01 16:53:16,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 134 (ObjNum 1608 snap 0 seq_num 1608) dirty exists 2014-07-01 16:53:16,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: finishing write tid 4 to mira0435601-161 2014-07-01 16:53:16,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 131 (ObjNum 1606 snap 0 seq_num 1606) dirty exists 2014-07-01 16:53:16,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 1 to mira0435601-219 2014-07-01 16:53:16,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 3 to mira0435601-219 2014-07-01 16:53:16,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: expect (ObjNum 618 snap 0 seq_num 618) 2014-07-01 16:53:16,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:625: done (5 left) 2014-07-01 16:53:16,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:627: done (4 left) 2014-07-01 16:53:16,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:628: done (3 left) 2014-07-01 16:53:16,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: write oid 96 current snap is 24 2014-07-01 16:53:16,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: seq_num 551 ranges {442405=548448,1549974=92431} 2014-07-01 16:53:16,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: writing mira0437250-96 from 442405 to 990853 tid 1 2014-07-01 16:53:16,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: writing mira0437250-96 from 1549974 to 1642405 tid 2 2014-07-01 16:53:16,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:630: snap_create 2014-07-01 16:53:16,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 1 to mira0437250-96 2014-07-01 16:53:16,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 2 to mira0437250-96 2014-07-01 16:53:16,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 4 to mira0437250-96 2014-07-01 16:53:16,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: finishing write tid 4 to mira0437250-49 2014-07-01 16:53:16,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 49 v 132 (ObjNum 548 snap 21 seq_num 548) dirty exists 2014-07-01 16:53:16,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: finishing write tid 4 to mira0437250-337 2014-07-01 16:53:16,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 107 (ObjNum 549 snap 23 seq_num 549) dirty exists 2014-07-01 16:53:16,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: finishing write tid 5 to mira0437250-96 2014-07-01 16:53:16,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 104 (ObjNum 551 snap 24 seq_num 551) dirty exists 2014-07-01 16:53:17,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: finishing write tid 1 to mira0437250-452 2014-07-01 16:53:17,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: finishing write tid 3 to mira0437250-452 2014-07-01 16:53:17,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: finishing write tid 4 to mira0437250-452 2014-07-01 16:53:17,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 176 (ObjNum 550 snap 23 seq_num 550) dirty exists 2014-07-01 16:53:17,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 4 to mira0435601-219 2014-07-01 16:53:17,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 292 (ObjNum 1604 snap 0 seq_num 1604) dirty exists 2014-07-01 16:53:17,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2492: done (10 left) 2014-07-01 16:53:17,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: done (9 left) 2014-07-01 16:53:17,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: done (8 left) 2014-07-01 16:53:17,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: done (7 left) 2014-07-01 16:53:17,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: done (6 left) 2014-07-01 16:53:17,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: done (5 left) 2014-07-01 16:53:17,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: read oid 77 snap -1 2014-07-01 16:53:17,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 358 current snap is 0 2014-07-01 16:53:17,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: seq_num 1610 ranges {2588672=585728,3174400=675840,3850240=126976} 2014-07-01 16:53:17,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: writing mira0435601-358 from 2588672 to 3174400 tid 1 2014-07-01 16:53:17,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: writing mira0435601-358 from 3174400 to 3850240 tid 2 2014-07-01 16:53:17,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: writing mira0435601-358 from 3850240 to 3977216 tid 3 2014-07-01 16:53:17,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2518: delete oid 63 current snap is 0 2014-07-01 16:53:17,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: expect (ObjNum 1596 snap 0 seq_num 1596) 2014-07-01 16:53:17,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:621: done (4 left) 2014-07-01 16:53:17,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:624: done (3 left) 2014-07-01 16:53:17,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:626: done (2 left) 2014-07-01 16:53:17,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:629: done (1 left) 2014-07-01 16:53:17,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:630: done (0 left) 2014-07-01 16:53:17,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: write oid 180 current snap is 25 2014-07-01 16:53:17,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: seq_num 552 ranges {787235=520252,2028010=359225} 2014-07-01 16:53:17,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: writing mira0437250-180 from 787235 to 1307487 tid 1 2014-07-01 16:53:17,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: writing mira0437250-180 from 2028010 to 2387235 tid 2 2014-07-01 16:53:17,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: read oid 183 snap -1 2014-07-01 16:53:17,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: snap_remove snap 15 2014-07-01 16:53:17,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 16:53:18,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 1 to mira0437250-180 2014-07-01 16:53:18,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 2 to mira0437250-180 2014-07-01 16:53:18,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 4 to mira0437250-180 2014-07-01 16:53:18,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: finishing write tid 5 to mira0437250-180 2014-07-01 16:53:18,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 188 (ObjNum 552 snap 25 seq_num 552) dirty exists 2014-07-01 16:53:18,814.814 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:18,814.814 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:18,815.815 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2014-07-01 16:53:18,815.815 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 1' 2014-07-01 16:53:18,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:631: done (2 left) 2014-07-01 16:53:18,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:632: done (1 left) 2014-07-01 16:53:18,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:633: done (0 left) 2014-07-01 16:53:18,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: rollback oid 476 current snap is 25 2014-07-01 16:53:18,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 476 to 24 2014-07-01 16:53:18,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:635: delete oid 390 current snap is 25 2014-07-01 16:53:18,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:635: done (1 left) 2014-07-01 16:53:18,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: write oid 202 current snap is 25 2014-07-01 16:53:18,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: seq_num 554 ranges {427294=615285,1820178=671603,2827293=1} 2014-07-01 16:53:18,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0437250-202 from 427294 to 1042579 tid 1 2014-07-01 16:53:18,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0437250-202 from 1820178 to 2491781 tid 2 2014-07-01 16:53:18,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: writing mira0437250-202 from 2827293 to 2827294 tid 3 2014-07-01 16:53:18,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: read oid 139 snap -1 2014-07-01 16:53:18,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 180 (ObjNum 475 snap 0 seq_num 475) dirty exists 2014-07-01 16:53:18,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 1 to mira0437250-202 2014-07-01 16:53:18,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:634: done (2 left) 2014-07-01 16:53:18,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:638: snap_remove snap 24 2014-07-01 16:53:19,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 2 to mira0437250-202 2014-07-01 16:53:19,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 3 to mira0437250-202 2014-07-01 16:53:19,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 5 to mira0437250-202 2014-07-01 16:53:19,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: finishing write tid 6 to mira0437250-202 2014-07-01 16:53:19,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 126 (ObjNum 554 snap 25 seq_num 554) dirty exists 2014-07-01 16:53:19,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: expect (ObjNum 138 snap 0 seq_num 138) 2014-07-01 16:53:19,832.832 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 1 (8655362) 2014-07-01 16:53:19,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: done (7 left) 2014-07-01 16:53:19,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2518: done (6 left) 2014-07-01 16:53:19,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: read oid 384 snap -1 2014-07-01 16:53:19,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2520: read oid 498 snap -1 2014-07-01 16:53:19,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 248 current snap is 0 2014-07-01 16:53:19,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: seq_num 1612 ranges {0=733184,733184=401408} 2014-07-01 16:53:20,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0435601-248 from 0 to 733184 tid 1 2014-07-01 16:53:20,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0435601-248 from 733184 to 1134592 tid 2 2014-07-01 16:53:20,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:2522: read oid 302 snap -1 2014-07-01 16:53:20,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:2520: done (9 left) 2014-07-01 16:53:20,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 244 current snap is 0 2014-07-01 16:53:20,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: seq_num 1613 ranges {1327104=757760,2084864=425984,2510848=606208,3117056=573440} 2014-07-01 16:53:20,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0435601-244 from 1327104 to 2084864 tid 1 2014-07-01 16:53:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0435601-244 from 2084864 to 2510848 tid 2 2014-07-01 16:53:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0435601-244 from 2510848 to 3117056 tid 3 2014-07-01 16:53:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0435601-244 from 3117056 to 3690496 tid 4 2014-07-01 16:53:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 93 current snap is 0 2014-07-01 16:53:20,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: seq_num 1614 ranges {0=434176,434176=659456,1093632=479232,1572864=466944} 2014-07-01 16:53:20,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: writing mira0435601-93 from 0 to 434176 tid 1 2014-07-01 16:53:20,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: writing mira0435601-93 from 434176 to 1093632 tid 2 2014-07-01 16:53:20,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: writing mira0435601-93 from 1093632 to 1572864 tid 3 2014-07-01 16:53:20,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: writing mira0435601-93 from 1572864 to 2039808 tid 4 2014-07-01 16:53:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 285 current snap is 0 2014-07-01 16:53:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: seq_num 1615 ranges {143360=708608,851968=643072,1495040=552960,2048000=8192} 2014-07-01 16:53:20,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: writing mira0435601-285 from 143360 to 851968 tid 1 2014-07-01 16:53:20,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: writing mira0435601-285 from 851968 to 1495040 tid 2 2014-07-01 16:53:20,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: writing mira0435601-285 from 1495040 to 2048000 tid 3 2014-07-01 16:53:20,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: writing mira0435601-285 from 2048000 to 2056192 tid 4 2014-07-01 16:53:20,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:2526: delete oid 314 current snap is 0 2014-07-01 16:53:20,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:636: done (2 left) 2014-07-01 16:53:20,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:637: done (1 left) 2014-07-01 16:53:20,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:638: done (0 left) 2014-07-01 16:53:20,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: delete oid 370 current snap is 25 2014-07-01 16:53:20,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 1 to mira0435601-93 2014-07-01 16:53:20,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 2 to mira0435601-93 2014-07-01 16:53:20,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 3 to mira0435601-93 2014-07-01 16:53:20,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 4 to mira0435601-93 2014-07-01 16:53:20,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 6 to mira0435601-93 2014-07-01 16:53:20,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: finishing write tid 7 to mira0435601-93 2014-07-01 16:53:20,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 113 (ObjNum 1614 snap 0 seq_num 1614) dirty exists 2014-07-01 16:53:21,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: finishing write tid 1 to mira0435601-358 2014-07-01 16:53:21,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: finishing write tid 2 to mira0435601-358 2014-07-01 16:53:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: finishing write tid 3 to mira0435601-358 2014-07-01 16:53:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: finishing write tid 5 to mira0435601-358 2014-07-01 16:53:21,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: finishing write tid 6 to mira0435601-358 2014-07-01 16:53:21,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 223 (ObjNum 1610 snap 0 seq_num 1610) dirty exists 2014-07-01 16:53:21,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:639: done (0 left) 2014-07-01 16:53:21,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: write oid 434 current snap is 25 2014-07-01 16:53:21,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: seq_num 556 ranges {582682=443566,1495995=714841,2758379=577829,3782681=1} 2014-07-01 16:53:21,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: writing mira0437250-434 from 582682 to 1026248 tid 1 2014-07-01 16:53:21,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: writing mira0437250-434 from 1495995 to 2210836 tid 2 2014-07-01 16:53:21,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: writing mira0437250-434 from 2758379 to 3336208 tid 3 2014-07-01 16:53:21,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: writing mira0437250-434 from 3782681 to 3782682 tid 4 2014-07-01 16:53:21,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: write oid 190 current snap is 25 2014-07-01 16:53:21,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: seq_num 557 ranges {527510=1} 2014-07-01 16:53:21,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: writing mira0437250-190 from 527510 to 527511 tid 1 2014-07-01 16:53:21,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: read oid 290 snap -1 2014-07-01 16:53:21,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: snap_remove snap 22 2014-07-01 16:53:21,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 1 to mira0435601-285 2014-07-01 16:53:21,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 2 to mira0435601-285 2014-07-01 16:53:21,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 3 to mira0435601-285 2014-07-01 16:53:21,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 4 to mira0435601-285 2014-07-01 16:53:21,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 6 to mira0435601-285 2014-07-01 16:53:21,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 16:53:21,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: expect (ObjNum 862 snap 0 seq_num 862) 2014-07-01 16:53:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 1 to mira0437250-190 2014-07-01 16:53:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 3 to mira0437250-190 2014-07-01 16:53:21,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: finishing write tid 4 to mira0437250-190 2014-07-01 16:53:21,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 118 (ObjNum 557 snap 25 seq_num 557) dirty exists 2014-07-01 16:53:22,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 1 to mira0437250-434 2014-07-01 16:53:22,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 2 to mira0437250-434 2014-07-01 16:53:22,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 3 to mira0437250-434 2014-07-01 16:53:22,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 4 to mira0437250-434 2014-07-01 16:53:22,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 6 to mira0437250-434 2014-07-01 16:53:22,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: finishing write tid 7 to mira0437250-434 2014-07-01 16:53:22,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 134 (ObjNum 556 snap 25 seq_num 556) dirty exists 2014-07-01 16:53:22,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:640: done (3 left) 2014-07-01 16:53:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:641: done (2 left) 2014-07-01 16:53:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:642: done (1 left) 2014-07-01 16:53:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:643: done (0 left) 2014-07-01 16:53:22,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: delete oid 496 current snap is 25 2014-07-01 16:53:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: finishing write tid 7 to mira0435601-285 2014-07-01 16:53:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 136 (ObjNum 1615 snap 0 seq_num 1615) dirty exists 2014-07-01 16:53:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 1 to mira0435601-248 2014-07-01 16:53:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 2 to mira0435601-248 2014-07-01 16:53:23,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 4 to mira0435601-248 2014-07-01 16:53:23,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2522: expect (ObjNum 986 snap 0 seq_num 986) 2014-07-01 16:53:24,844.844 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:24,845.845 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:24,845.845 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2014-07-01 16:53:24,846.846 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 1' 2014-07-01 16:53:25,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:644: done (0 left) 2014-07-01 16:53:25,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: read oid 284 snap 20 2014-07-01 16:53:25,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:646: rollback oid 420 current snap is 25 2014-07-01 16:53:25,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 420 to 21 2014-07-01 16:53:25,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: snap_remove snap 17 2014-07-01 16:53:25,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: expect (ObjNum 283 snap 0 seq_num 283) 2014-07-01 16:53:25,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 239 (ObjNum 419 snap 0 seq_num 419) dirty exists 2014-07-01 16:53:25,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:645: done (2 left) 2014-07-01 16:53:25,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:646: done (1 left) 2014-07-01 16:53:25,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:647: done (0 left) 2014-07-01 16:53:25,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: read oid 341 snap 21 2014-07-01 16:53:25,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: read oid 413 snap 1 2014-07-01 16:53:25,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: read oid 372 snap -1 2014-07-01 16:53:25,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: write oid 170 current snap is 25 2014-07-01 16:53:25,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: seq_num 559 ranges {683923=546499,1483922=1} 2014-07-01 16:53:25,527.527 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 1 (8655362) 2014-07-01 16:53:25,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: writing mira0437250-170 from 683923 to 1230422 tid 1 2014-07-01 16:53:25,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: writing mira0437250-170 from 1483922 to 1483923 tid 2 2014-07-01 16:53:25,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: read oid 492 snap -1 2014-07-01 16:53:25,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:653: snap_create 2014-07-01 16:53:25,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: expect (ObjNum 340 snap 0 seq_num 340) 2014-07-01 16:53:26,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: expect (ObjNum 412 snap 0 seq_num 412) 2014-07-01 16:53:26,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: expect (ObjNum 491 snap 0 seq_num 491) 2014-07-01 16:53:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 5 to mira0435601-248 2014-07-01 16:53:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 140 (ObjNum 1612 snap 0 seq_num 1612) dirty exists 2014-07-01 16:53:27,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: expect (ObjNum 1144 snap 0 seq_num 1144) 2014-07-01 16:53:28,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: expect (ObjNum 371 snap 0 seq_num 371) 2014-07-01 16:53:28,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: finishing write tid 1 to mira0437250-170 2014-07-01 16:53:28,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: finishing write tid 2 to mira0437250-170 2014-07-01 16:53:28,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: finishing write tid 4 to mira0437250-170 2014-07-01 16:53:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: finishing write tid 5 to mira0437250-170 2014-07-01 16:53:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 122 (ObjNum 559 snap 25 seq_num 559) dirty exists 2014-07-01 16:53:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:648: done (5 left) 2014-07-01 16:53:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:649: done (4 left) 2014-07-01 16:53:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:650: done (3 left) 2014-07-01 16:53:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:651: done (2 left) 2014-07-01 16:53:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:652: done (1 left) 2014-07-01 16:53:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:653: done (0 left) 2014-07-01 16:53:28,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: snap_remove snap 23 2014-07-01 16:53:28,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:654: done (0 left) 2014-07-01 16:53:28,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: write oid 120 current snap is 26 2014-07-01 16:53:28,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: seq_num 560 ranges {627728=518303,1880330=347398} 2014-07-01 16:53:28,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: writing mira0437250-120 from 627728 to 1146031 tid 1 2014-07-01 16:53:28,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: writing mira0437250-120 from 1880330 to 2227728 tid 2 2014-07-01 16:53:28,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: write oid 27 current snap is 26 2014-07-01 16:53:28,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: seq_num 561 ranges {572558=642555,1844062=700889,2972557=1} 2014-07-01 16:53:28,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: writing mira0437250-27 from 572558 to 1215113 tid 1 2014-07-01 16:53:28,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: writing mira0437250-27 from 1844062 to 2544951 tid 2 2014-07-01 16:53:28,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: writing mira0437250-27 from 2972557 to 2972558 tid 3 2014-07-01 16:53:28,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: write oid 221 current snap is 26 2014-07-01 16:53:28,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: seq_num 562 ranges {668812=615384,2075571=593241} 2014-07-01 16:53:28,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: writing mira0437250-221 from 668812 to 1284196 tid 1 2014-07-01 16:53:28,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: writing mira0437250-221 from 2075571 to 2668812 tid 2 2014-07-01 16:53:29,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: write oid 6 current snap is 26 2014-07-01 16:53:29,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: seq_num 563 ranges {612618=622260,1672327=592703,2949803=462815} 2014-07-01 16:53:29,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: writing mira0437250-6 from 612618 to 1234878 tid 1 2014-07-01 16:53:29,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 1 to mira0435601-244 2014-07-01 16:53:29,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 2 to mira0435601-244 2014-07-01 16:53:29,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 3 to mira0435601-244 2014-07-01 16:53:29,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 4 to mira0435601-244 2014-07-01 16:53:29,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 6 to mira0435601-244 2014-07-01 16:53:29,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 7 to mira0435601-244 2014-07-01 16:53:29,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 201 (ObjNum 1613 snap 0 seq_num 1613) dirty exists 2014-07-01 16:53:29,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: done (12 left) 2014-07-01 16:53:29,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: done (11 left) 2014-07-01 16:53:29,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: done (10 left) 2014-07-01 16:53:29,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: done (9 left) 2014-07-01 16:53:29,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2522: done (8 left) 2014-07-01 16:53:29,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: done (7 left) 2014-07-01 16:53:29,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: done (6 left) 2014-07-01 16:53:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: done (5 left) 2014-07-01 16:53:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2526: done (4 left) 2014-07-01 16:53:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 16:53:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: seq_num 1617 ranges {2715648=606208} 2014-07-01 16:53:29,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: writing mira0437250-6 from 1672327 to 2265030 tid 2 2014-07-01 16:53:29,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: writing mira0437250-6 from 2949803 to 3412618 tid 3 2014-07-01 16:53:29,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:659: rollback oid 385 current snap is 26 2014-07-01 16:53:29,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 385 to 9 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:660: rollback oid 180 current snap is 26 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 180 to 20 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: write oid 366 current snap is 26 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: seq_num 564 ranges {424199=594064,1466714=662898,2714931=507865,3624198=1} 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: writing mira0435601-397 from 2715648 to 3321856 tid 1 2014-07-01 16:53:29,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2528: delete oid 346 current snap is 0 2014-07-01 16:53:29,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0437250-366 from 424199 to 1018263 tid 1 2014-07-01 16:53:29,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0437250-366 from 1466714 to 2129612 tid 2 2014-07-01 16:53:29,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0437250-366 from 2714931 to 3222796 tid 3 2014-07-01 16:53:29,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: writing mira0437250-366 from 3624198 to 3624199 tid 4 2014-07-01 16:53:29,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: read oid 52 snap -1 2014-07-01 16:53:29,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: rollback oid 487 current snap is 26 2014-07-01 16:53:29,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 487 to 18 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:664: read oid 240 snap -1 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: write oid 322 current snap is 26 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: seq_num 565 ranges {368004=1} 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: writing mira0437250-322 from 368004 to 368005 tid 1 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: rollback oid 66 current snap is 26 2014-07-01 16:53:29,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 66 to 20 2014-07-01 16:53:29,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:667: snap_create 2014-07-01 16:53:29,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: expect (ObjNum 51 snap 0 seq_num 51) 2014-07-01 16:53:29,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:664: expect (ObjNum 239 snap 0 seq_num 239) 2014-07-01 16:53:29,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2512: expect (ObjNum 48 snap 0 seq_num 48) 2014-07-01 16:53:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 1 to mira0437250-366 2014-07-01 16:53:29,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 2 to mira0437250-366 2014-07-01 16:53:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 3 to mira0437250-366 2014-07-01 16:53:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 4 to mira0437250-366 2014-07-01 16:53:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 6 to mira0437250-366 2014-07-01 16:53:29,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: finishing write tid 7 to mira0437250-366 2014-07-01 16:53:29,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 118 (ObjNum 564 snap 26 seq_num 564) dirty exists 2014-07-01 16:53:29,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: finishing write tid 1 to mira0437250-120 2014-07-01 16:53:29,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: finishing write tid 2 to mira0437250-120 2014-07-01 16:53:29,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: finishing write tid 4 to mira0437250-120 2014-07-01 16:53:29,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: finishing write tid 5 to mira0437250-120 2014-07-01 16:53:29,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 113 (ObjNum 560 snap 26 seq_num 560) dirty exists 2014-07-01 16:53:29,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:655: done (12 left) 2014-07-01 16:53:29,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:661: done (11 left) 2014-07-01 16:53:29,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:662: done (10 left) 2014-07-01 16:53:29,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:664: done (9 left) 2014-07-01 16:53:29,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:667: done (8 left) 2014-07-01 16:53:29,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: write oid 13 current snap is 27 2014-07-01 16:53:29,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: seq_num 566 ranges {712835=400000} 2014-07-01 16:53:29,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: writing mira0437250-13 from 712835 to 1112835 tid 1 2014-07-01 16:53:29,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: read oid 441 snap 9 2014-07-01 16:53:29,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:670: read oid 19 snap 26 2014-07-01 16:53:29,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: read oid 67 snap -1 2014-07-01 16:53:29,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: read oid 485 snap 16 2014-07-01 16:53:29,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:673: delete oid 135 current snap is 27 2014-07-01 16:53:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 1 to mira0437250-27 2014-07-01 16:53:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 2 to mira0437250-27 2014-07-01 16:53:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 3 to mira0437250-27 2014-07-01 16:53:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 5 to mira0437250-27 2014-07-01 16:53:30,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: finishing write tid 1 to mira0437250-6 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: finishing write tid 2 to mira0437250-6 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: finishing write tid 3 to mira0437250-6 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: finishing write tid 5 to mira0437250-6 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 205 (ObjNum 179 snap 0 seq_num 179) dirty exists 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: expect (ObjNum 440 snap 0 seq_num 440) 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2512: done (5 left) 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2528: done (4 left) 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 166 current snap is 0 2014-07-01 16:53:30,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: seq_num 1619 ranges {3887104=798720,4685824=401408} 2014-07-01 16:53:30,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: writing mira0435601-166 from 3887104 to 4685824 tid 1 2014-07-01 16:53:30,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: writing mira0435601-166 from 4685824 to 5087232 tid 2 2014-07-01 16:53:30,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: read oid 298 snap -1 2014-07-01 16:53:30,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 239 current snap is 0 2014-07-01 16:53:30,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: seq_num 1620 ranges {0=24576} 2014-07-01 16:53:30,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: writing mira0435601-239 from 0 to 24576 tid 1 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2532: read oid 178 snap -1 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2533: read oid 160 snap -1 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2534: read oid 365 snap -1 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2535: read oid 412 snap -1 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 346 current snap is 0 2014-07-01 16:53:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: seq_num 1621 ranges {0=696320,696320=634880,1331200=495616,1826816=475136} 2014-07-01 16:53:30,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: writing mira0435601-346 from 0 to 696320 tid 1 2014-07-01 16:53:30,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: writing mira0435601-346 from 696320 to 1331200 tid 2 2014-07-01 16:53:30,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: writing mira0435601-346 from 1331200 to 1826816 tid 3 2014-07-01 16:53:30,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: writing mira0435601-346 from 1826816 to 2301952 tid 4 2014-07-01 16:53:30,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: read oid 102 snap -1 2014-07-01 16:53:30,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:2534: expect (ObjNum 364 snap 0 seq_num 364) 2014-07-01 16:53:30,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2535: expect (ObjNum 1053 snap 0 seq_num 1053) 2014-07-01 16:53:30,529.529 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:30,529.529 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:30,529.529 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.180207 2014-07-01 16:53:30,529.529 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 0.180207070735' 2014-07-01 16:53:30,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 1 to mira0437250-322 2014-07-01 16:53:30,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 3 to mira0437250-322 2014-07-01 16:53:30,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: finishing write tid 4 to mira0437250-322 2014-07-01 16:53:30,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 116 (ObjNum 565 snap 26 seq_num 565) dirty exists 2014-07-01 16:53:30,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:53:31,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: finishing write tid 6 to mira0437250-27 2014-07-01 16:53:31,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 122 (ObjNum 561 snap 26 seq_num 561) dirty exists 2014-07-01 16:53:31,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: expect (ObjNum 66 snap 0 seq_num 66) 2014-07-01 16:53:31,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:670: expect (ObjNum 18 snap 0 seq_num 18) 2014-07-01 16:53:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: finishing write tid 6 to mira0437250-6 2014-07-01 16:53:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 203 (ObjNum 563 snap 26 seq_num 563) dirty exists 2014-07-01 16:53:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 118 (ObjNum 65 snap 0 seq_num 65) dirty exists 2014-07-01 16:53:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: finishing write tid 1 to mira0437250-13 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: finishing write tid 3 to mira0437250-13 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:656: done (13 left) 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:658: done (12 left) 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:660: done (11 left) 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:665: done (10 left) 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:666: done (9 left) 2014-07-01 16:53:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:669: done (8 left) 2014-07-01 16:53:31,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:670: done (7 left) 2014-07-01 16:53:31,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:671: done (6 left) 2014-07-01 16:53:31,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:672: done (5 left) 2014-07-01 16:53:31,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:673: done (4 left) 2014-07-01 16:53:31,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: write oid 497 current snap is 27 2014-07-01 16:53:31,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: seq_num 568 ranges {752894=517378,1552893=1} 2014-07-01 16:53:31,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: writing mira0437250-497 from 752894 to 1270272 tid 1 2014-07-01 16:53:31,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: writing mira0437250-497 from 1552893 to 1552894 tid 2 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: finishing write tid 4 to mira0437250-13 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 131 (ObjNum 566 snap 27 seq_num 566) dirty exists 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 385 v 136 (ObjNum 384 snap 0 seq_num 384) dirty exists 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 172 (ObjNum 486 snap 0 seq_num 486) dirty exists 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: read oid 371 snap -1 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:659: done (5 left) 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:663: done (4 left) 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:668: done (3 left) 2014-07-01 16:53:31,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:676: rollback oid 5 current snap is 27 2014-07-01 16:53:31,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 5 to 25 2014-07-01 16:53:31,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:677: snap_remove snap 1 2014-07-01 16:53:31,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: expect (ObjNum 370 snap 0 seq_num 370) 2014-07-01 16:53:31,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: expect (ObjNum 1302 snap 0 seq_num 1302) 2014-07-01 16:53:31,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 133 (ObjNum 4 snap 0 seq_num 4) dirty exists 2014-07-01 16:53:31,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: finishing write tid 1 to mira0437250-221 2014-07-01 16:53:31,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: finishing write tid 2 to mira0437250-221 2014-07-01 16:53:31,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: finishing write tid 4 to mira0437250-221 2014-07-01 16:53:31,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: finishing write tid 5 to mira0437250-221 2014-07-01 16:53:31,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 176 (ObjNum 562 snap 26 seq_num 562) dirty exists 2014-07-01 16:53:31,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:657: done (4 left) 2014-07-01 16:53:31,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:675: done (3 left) 2014-07-01 16:53:31,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:676: done (2 left) 2014-07-01 16:53:31,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:677: done (1 left) 2014-07-01 16:53:31,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: read oid 415 snap -1 2014-07-01 16:53:31,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:679: delete oid 221 current snap is 27 2014-07-01 16:53:31,913.913 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 0.180207 (8118102) 2014-07-01 16:53:31,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: expect (ObjNum 414 snap 0 seq_num 414) 2014-07-01 16:53:33,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:678: done (2 left) 2014-07-01 16:53:33,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:679: done (1 left) 2014-07-01 16:53:33,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: write oid 147 current snap is 27 2014-07-01 16:53:33,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: seq_num 570 ranges {509306=613434,1776468=621990,2509305=1} 2014-07-01 16:53:33,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: writing mira0437250-147 from 509306 to 1122740 tid 1 2014-07-01 16:53:33,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: writing mira0437250-147 from 1776468 to 2398458 tid 2 2014-07-01 16:53:33,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: writing mira0437250-147 from 2509305 to 2509306 tid 3 2014-07-01 16:53:33,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: write oid 37 current snap is 27 2014-07-01 16:53:33,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: seq_num 571 ranges {454136=620311,1774249=540761,3037559=216577} 2014-07-01 16:53:33,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: writing mira0437250-37 from 454136 to 1074447 tid 1 2014-07-01 16:53:33,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: writing mira0437250-37 from 1774249 to 2315010 tid 2 2014-07-01 16:53:33,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: writing mira0437250-37 from 3037559 to 3254136 tid 3 2014-07-01 16:53:33,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: snap_create 2014-07-01 16:53:33,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: finishing write tid 1 to mira0437250-147 2014-07-01 16:53:33,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: finishing write tid 1 to mira0437250-497 2014-07-01 16:53:33,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: finishing write tid 2 to mira0437250-497 2014-07-01 16:53:33,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: finishing write tid 4 to mira0437250-497 2014-07-01 16:53:33,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: finishing write tid 5 to mira0437250-497 2014-07-01 16:53:33,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 196 (ObjNum 568 snap 27 seq_num 568) dirty exists 2014-07-01 16:53:33,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: finishing write tid 1 to mira0437250-37 2014-07-01 16:53:33,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: finishing write tid 2 to mira0437250-37 2014-07-01 16:53:33,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: finishing write tid 3 to mira0437250-37 2014-07-01 16:53:33,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: finishing write tid 5 to mira0437250-37 2014-07-01 16:53:33,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: finishing write tid 6 to mira0437250-37 2014-07-01 16:53:33,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 159 (ObjNum 571 snap 27 seq_num 571) dirty exists 2014-07-01 16:53:33,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: finishing write tid 2 to mira0437250-147 2014-07-01 16:53:33,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: finishing write tid 3 to mira0437250-147 2014-07-01 16:53:33,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: finishing write tid 5 to mira0437250-147 2014-07-01 16:53:33,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: finishing write tid 6 to mira0437250-147 2014-07-01 16:53:33,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 127 (ObjNum 570 snap 27 seq_num 570) dirty exists 2014-07-01 16:53:33,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:674: done (3 left) 2014-07-01 16:53:33,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:680: done (2 left) 2014-07-01 16:53:33,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:681: done (1 left) 2014-07-01 16:53:33,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:682: done (0 left) 2014-07-01 16:53:33,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: read oid 267 snap -1 2014-07-01 16:53:33,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: rollback oid 212 current snap is 28 2014-07-01 16:53:33,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 212 to 20 2014-07-01 16:53:33,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:685: rollback oid 134 current snap is 28 2014-07-01 16:53:33,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 134 to 27 2014-07-01 16:53:33,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:686: delete oid 20 current snap is 28 2014-07-01 16:53:33,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: expect (ObjNum 266 snap 0 seq_num 266) 2014-07-01 16:53:34,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: finishing write tid 1 to mira0435601-239 2014-07-01 16:53:34,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: finishing write tid 3 to mira0435601-239 2014-07-01 16:53:34,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: finishing write tid 1 to mira0435601-166 2014-07-01 16:53:34,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: finishing write tid 4 to mira0435601-239 2014-07-01 16:53:34,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 142 (ObjNum 1620 snap 0 seq_num 1620) dirty exists 2014-07-01 16:53:34,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 1 to mira0435601-346 2014-07-01 16:53:34,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: done (12 left) 2014-07-01 16:53:34,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: done (11 left) 2014-07-01 16:53:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2532: done (10 left) 2014-07-01 16:53:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2534: done (9 left) 2014-07-01 16:53:34,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2535: done (8 left) 2014-07-01 16:53:34,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 341 current snap is 0 2014-07-01 16:53:34,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: seq_num 1622 ranges {0=569344,569344=462848,1032192=774144,1806336=368640} 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 132 (ObjNum 133 snap 0 seq_num 133) dirty exists 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 128 (ObjNum 211 snap 0 seq_num 211) dirty exists 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:683: done (3 left) 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:684: done (2 left) 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:685: done (1 left) 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:686: done (0 left) 2014-07-01 16:53:34,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: delete oid 479 current snap is 28 2014-07-01 16:53:34,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: writing mira0435601-341 from 0 to 569344 tid 1 2014-07-01 16:53:34,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:687: done (0 left) 2014-07-01 16:53:34,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: read oid 373 snap 25 2014-07-01 16:53:34,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: read oid 200 snap 16 2014-07-01 16:53:34,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:690: rollback oid 254 current snap is 28 2014-07-01 16:53:34,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 254 to 27 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: delete oid 190 current snap is 28 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:691: done (3 left) 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: read oid 485 snap -1 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: write oid 441 current snap is 28 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: seq_num 575 ranges {498159=695048,1698158=1} 2014-07-01 16:53:34,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: writing mira0437250-441 from 498159 to 1193207 tid 1 2014-07-01 16:53:34,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: writing mira0437250-441 from 1698158 to 1698159 tid 2 2014-07-01 16:53:34,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: read oid 360 snap -1 2014-07-01 16:53:34,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 130 (ObjNum 253 snap 0 seq_num 253) dirty exists 2014-07-01 16:53:34,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: expect (ObjNum 484 snap 0 seq_num 484) 2014-07-01 16:53:34,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: writing mira0435601-341 from 569344 to 1032192 tid 2 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: writing mira0435601-341 from 1032192 to 1806336 tid 3 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: writing mira0435601-341 from 1806336 to 2174976 tid 4 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: seq_num 1623 ranges {4608000=647168,5255168=446464,5701632=598016,6299648=159744} 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0435601-11 from 4608000 to 5255168 tid 1 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0435601-11 from 5255168 to 5701632 tid 2 2014-07-01 16:53:34,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0435601-11 from 5701632 to 6299648 tid 3 2014-07-01 16:53:34,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0435601-11 from 6299648 to 6459392 tid 4 2014-07-01 16:53:34,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 500 current snap is 0 2014-07-01 16:53:34,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: seq_num 1624 ranges {0=516096,516096=630784,1146880=573440} 2014-07-01 16:53:34,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: writing mira0435601-500 from 0 to 516096 tid 1 2014-07-01 16:53:34,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: writing mira0435601-500 from 516096 to 1146880 tid 2 2014-07-01 16:53:34,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: writing mira0435601-500 from 1146880 to 1720320 tid 3 2014-07-01 16:53:34,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 19 current snap is 0 2014-07-01 16:53:34,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: seq_num 1625 ranges {0=790528,790528=614400,1404928=188416} 2014-07-01 16:53:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: writing mira0435601-19 from 0 to 790528 tid 1 2014-07-01 16:53:34,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: expect (ObjNum 372 snap 0 seq_num 372) 2014-07-01 16:53:34,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: writing mira0435601-19 from 790528 to 1404928 tid 2 2014-07-01 16:53:34,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: writing mira0435601-19 from 1404928 to 1593344 tid 3 2014-07-01 16:53:34,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2542: delete oid 134 current snap is 0 2014-07-01 16:53:34,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 2 to mira0435601-346 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 3 to mira0435601-346 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: finishing write tid 2 to mira0435601-166 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: finishing write tid 4 to mira0435601-166 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: finishing write tid 5 to mira0435601-166 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 264 (ObjNum 1619 snap 0 seq_num 1619) dirty exists 2014-07-01 16:53:34,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: expect (ObjNum 1253 snap 0 seq_num 1253) 2014-07-01 16:53:35,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: finishing write tid 1 to mira0437250-441 2014-07-01 16:53:35,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: finishing write tid 2 to mira0437250-441 2014-07-01 16:53:35,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: finishing write tid 4 to mira0437250-441 2014-07-01 16:53:35,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: finishing write tid 5 to mira0437250-441 2014-07-01 16:53:35,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 441 v 134 (ObjNum 575 snap 28 seq_num 575) dirty exists 2014-07-01 16:53:35,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: expect (ObjNum 199 snap 0 seq_num 199) 2014-07-01 16:53:35,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: expect (ObjNum 359 snap 0 seq_num 359) 2014-07-01 16:53:36,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 4 to mira0435601-346 2014-07-01 16:53:36,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 6 to mira0435601-346 2014-07-01 16:53:36,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: finishing write tid 7 to mira0435601-346 2014-07-01 16:53:36,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 249 (ObjNum 1621 snap 0 seq_num 1621) dirty exists 2014-07-01 16:53:36,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:2533: expect (ObjNum 814 snap 0 seq_num 814) 2014-07-01 16:53:36,921.921 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:36,922.922 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:36,922.922 INFO:teuthology.task.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2014-07-01 16:53:36,923.923 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 1 0' 2014-07-01 16:53:38,012.012 INFO:teuthology.orchestra.run.mira079.stderr:set osd.1 primary-affinity to 0 (802) 2014-07-01 16:53:38,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:688: done (5 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:689: done (4 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:690: done (3 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:692: done (2 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:693: done (1 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:694: done (0 left) 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:695: rollback oid 130 current snap is 28 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 130 to 19 2014-07-01 16:53:38,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:696: snap_remove snap 16 2014-07-01 16:53:38,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 145 (ObjNum 129 snap 0 seq_num 129) dirty exists 2014-07-01 16:53:38,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: finishing write tid 1 to mira0435601-397 2014-07-01 16:53:38,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: finishing write tid 3 to mira0435601-397 2014-07-01 16:53:38,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: finishing write tid 4 to mira0435601-397 2014-07-01 16:53:38,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 147 (ObjNum 1617 snap 0 seq_num 1617) dirty exists 2014-07-01 16:53:38,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 1 to mira0435601-153 2014-07-01 16:53:38,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 2 to mira0435601-153 2014-07-01 16:53:38,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 3 to mira0435601-153 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: done (12 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: done (11 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2533: done (10 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: done (9 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: done (8 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2542: done (7 left) 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: read oid 423 snap -1 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 4 to mira0435601-153 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 6 to mira0435601-153 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 7 to mira0435601-153 2014-07-01 16:53:38,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 175 (ObjNum 1599 snap 0 seq_num 1599) dirty exists 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: finishing write tid 1 to mira0435601-284 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: finishing write tid 2 to mira0435601-284 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: finishing write tid 3 to mira0435601-284 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: finishing write tid 5 to mira0435601-284 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: finishing write tid 6 to mira0435601-284 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 277 (ObjNum 1607 snap 0 seq_num 1607) dirty exists 2014-07-01 16:53:38,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: expect (ObjNum 1342 snap 0 seq_num 1342) 2014-07-01 16:53:39,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:695: done (1 left) 2014-07-01 16:53:39,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:696: done (0 left) 2014-07-01 16:53:39,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:697: delete oid 213 current snap is 28 2014-07-01 16:53:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:697: done (0 left) 2014-07-01 16:53:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: write oid 215 current snap is 28 2014-07-01 16:53:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: seq_num 577 ranges {690666=400000} 2014-07-01 16:53:39,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: writing mira0437250-215 from 690666 to 1090666 tid 1 2014-07-01 16:53:39,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:699: read oid 81 snap -1 2014-07-01 16:53:39,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: read oid 114 snap -1 2014-07-01 16:53:39,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: rollback oid 331 current snap is 28 2014-07-01 16:53:39,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 331 to 26 2014-07-01 16:53:39,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: rollback oid 496 current snap is 28 2014-07-01 16:53:39,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 496 to 19 2014-07-01 16:53:39,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: write oid 465 current snap is 28 2014-07-01 16:53:39,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: seq_num 578 ranges {599400=611485,1878389=569023,3154744=244656} 2014-07-01 16:53:39,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: writing mira0437250-465 from 599400 to 1210885 tid 1 2014-07-01 16:53:39,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: writing mira0437250-465 from 1878389 to 2447412 tid 2 2014-07-01 16:53:39,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: writing mira0437250-465 from 3154744 to 3399400 tid 3 2014-07-01 16:53:39,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: snap_create 2014-07-01 16:53:39,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 125 (ObjNum 495 snap 0 seq_num 495) dirty exists 2014-07-01 16:53:39,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: finishing write tid 1 to mira0437250-215 2014-07-01 16:53:39,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: finishing write tid 3 to mira0437250-215 2014-07-01 16:53:39,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: finishing write tid 4 to mira0437250-215 2014-07-01 16:53:39,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 228 (ObjNum 577 snap 28 seq_num 577) dirty exists 2014-07-01 16:53:39,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: expect (ObjNum 113 snap 0 seq_num 113) 2014-07-01 16:53:41,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: finishing write tid 1 to mira0437250-465 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: finishing write tid 2 to mira0437250-465 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: finishing write tid 3 to mira0437250-465 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: finishing write tid 5 to mira0437250-465 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: finishing write tid 6 to mira0437250-465 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 465 v 233 (ObjNum 578 snap 28 seq_num 578) dirty exists 2014-07-01 16:53:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 142 (ObjNum 330 snap 0 seq_num 330) dirty exists 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:698: done (6 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:699: done (5 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:700: done (4 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:701: done (3 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:702: done (2 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:703: done (1 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:704: done (0 left) 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: write oid 318 current snap is 29 2014-07-01 16:53:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: seq_num 579 ranges {695653=618362,1841097=639218,3089217=6436} 2014-07-01 16:53:41,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: writing mira0437250-318 from 695653 to 1314015 tid 1 2014-07-01 16:53:41,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: writing mira0437250-318 from 1841097 to 2480315 tid 2 2014-07-01 16:53:41,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: writing mira0437250-318 from 3089217 to 3095653 tid 3 2014-07-01 16:53:41,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: snap_remove snap 28 2014-07-01 16:53:41,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 1 to mira0437250-318 2014-07-01 16:53:41,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 2 to mira0437250-318 2014-07-01 16:53:41,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 3 to mira0437250-318 2014-07-01 16:53:41,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 5 to mira0437250-318 2014-07-01 16:53:41,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: finishing write tid 6 to mira0437250-318 2014-07-01 16:53:41,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 104 (ObjNum 579 snap 29 seq_num 579) dirty exists 2014-07-01 16:53:42,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:705: done (1 left) 2014-07-01 16:53:42,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:706: done (0 left) 2014-07-01 16:53:42,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:707: rollback oid 263 current snap is 29 2014-07-01 16:53:42,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 263 to 18 2014-07-01 16:53:42,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: snap_remove snap 25 2014-07-01 16:53:42,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 143 (ObjNum 262 snap 0 seq_num 262) dirty exists 2014-07-01 16:53:42,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 1 to mira0435601-341 2014-07-01 16:53:42,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 2 to mira0435601-341 2014-07-01 16:53:42,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 3 to mira0435601-341 2014-07-01 16:53:42,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 4 to mira0435601-341 2014-07-01 16:53:42,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 6 to mira0435601-341 2014-07-01 16:53:42,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: finishing write tid 7 to mira0435601-341 2014-07-01 16:53:42,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 245 (ObjNum 1622 snap 0 seq_num 1622) dirty exists 2014-07-01 16:53:42,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 1 to mira0435601-11 2014-07-01 16:53:42,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 2 to mira0435601-11 2014-07-01 16:53:42,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 3 to mira0435601-11 2014-07-01 16:53:42,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 4 to mira0435601-11 2014-07-01 16:53:42,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 6 to mira0435601-11 2014-07-01 16:53:42,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 7 to mira0435601-11 2014-07-01 16:53:42,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 141 (ObjNum 1623 snap 0 seq_num 1623) dirty exists 2014-07-01 16:53:42,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: finishing write tid 1 to mira0435601-500 2014-07-01 16:53:42,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: finishing write tid 2 to mira0435601-500 2014-07-01 16:53:42,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: finishing write tid 3 to mira0435601-500 2014-07-01 16:53:42,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: finishing write tid 5 to mira0435601-500 2014-07-01 16:53:42,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: done (7 left) 2014-07-01 16:53:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: done (6 left) 2014-07-01 16:53:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: done (5 left) 2014-07-01 16:53:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: done (4 left) 2014-07-01 16:53:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: done (3 left) 2014-07-01 16:53:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 377 current snap is 0 2014-07-01 16:53:42,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: seq_num 1627 ranges {4542464=286720} 2014-07-01 16:53:42,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: writing mira0435601-377 from 4542464 to 4829184 tid 1 2014-07-01 16:53:42,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 16:53:42,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: seq_num 1628 ranges {6160384=765952,6926336=454656,7380992=720896,8101888=430080} 2014-07-01 16:53:42,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0435601-409 from 6160384 to 6926336 tid 1 2014-07-01 16:53:42,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0435601-409 from 6926336 to 7380992 tid 2 2014-07-01 16:53:42,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0435601-409 from 7380992 to 8101888 tid 3 2014-07-01 16:53:42,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0435601-409 from 8101888 to 8531968 tid 4 2014-07-01 16:53:42,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 120 current snap is 0 2014-07-01 16:53:42,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: seq_num 1629 ranges {5316608=638976,5955584=438272,6393856=438272,6832128=729088} 2014-07-01 16:53:42,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0435601-120 from 5316608 to 5955584 tid 1 2014-07-01 16:53:42,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0435601-120 from 5955584 to 6393856 tid 2 2014-07-01 16:53:42,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0435601-120 from 6393856 to 6832128 tid 3 2014-07-01 16:53:42,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0435601-120 from 6832128 to 7561216 tid 4 2014-07-01 16:53:42,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 16:53:42,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: seq_num 1630 ranges {0=507904,507904=671744,1179648=667648,1847296=266240} 2014-07-01 16:53:42,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: writing mira0435601-418 from 0 to 507904 tid 1 2014-07-01 16:53:42,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: writing mira0435601-418 from 507904 to 1179648 tid 2 2014-07-01 16:53:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: writing mira0435601-418 from 1179648 to 1847296 tid 3 2014-07-01 16:53:42,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: writing mira0435601-418 from 1847296 to 2113536 tid 4 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: read oid 28 snap -1 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: finishing write tid 6 to mira0435601-500 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 500 v 200 (ObjNum 1624 snap 0 seq_num 1624) dirty exists 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: finishing write tid 1 to mira0435601-19 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: finishing write tid 2 to mira0435601-19 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: finishing write tid 3 to mira0435601-19 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: finishing write tid 5 to mira0435601-19 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: finishing write tid 6 to mira0435601-19 2014-07-01 16:53:42,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 145 (ObjNum 1625 snap 0 seq_num 1625) dirty exists 2014-07-01 16:53:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: done (7 left) 2014-07-01 16:53:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: done (6 left) 2014-07-01 16:53:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2549: read oid 414 snap -1 2014-07-01 16:53:42,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: expect (ObjNum 1556 snap 0 seq_num 1556) 2014-07-01 16:53:43,042.042 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:43,042.042 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:43,042.042 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2014-07-01 16:53:43,043.043 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0' 2014-07-01 16:53:43,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:707: done (1 left) 2014-07-01 16:53:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:708: done (0 left) 2014-07-01 16:53:43,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:709: delete oid 308 current snap is 29 2014-07-01 16:53:43,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:709: done (0 left) 2014-07-01 16:53:43,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:710: snap_create 2014-07-01 16:53:44,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:710: done (0 left) 2014-07-01 16:53:44,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:711: read oid 423 snap -1 2014-07-01 16:53:44,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:712: read oid 434 snap -1 2014-07-01 16:53:44,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:713: snap_remove snap 26 2014-07-01 16:53:44,105.105 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0 (802) 2014-07-01 16:53:44,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:712: expect (ObjNum 556 snap 25 seq_num 556) 2014-07-01 16:53:46,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:711: expect (ObjNum 422 snap 0 seq_num 422) 2014-07-01 16:53:46,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: finishing write tid 1 to mira0435601-377 2014-07-01 16:53:46,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: finishing write tid 3 to mira0435601-377 2014-07-01 16:53:46,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: finishing write tid 4 to mira0435601-377 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 255 (ObjNum 1627 snap 0 seq_num 1627) dirty exists 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 1 to mira0435601-409 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: done (6 left) 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: done (5 left) 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 253 current snap is 0 2014-07-01 16:53:47,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: seq_num 1631 ranges {3780608=536576,4317184=401408} 2014-07-01 16:53:47,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: writing mira0435601-253 from 3780608 to 4317184 tid 1 2014-07-01 16:53:47,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: writing mira0435601-253 from 4317184 to 4718592 tid 2 2014-07-01 16:53:47,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 107 current snap is 0 2014-07-01 16:53:47,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 2 to mira0435601-409 2014-07-01 16:53:47,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 3 to mira0435601-409 2014-07-01 16:53:47,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 4 to mira0435601-409 2014-07-01 16:53:47,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 6 to mira0435601-409 2014-07-01 16:53:47,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 7 to mira0435601-409 2014-07-01 16:53:47,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 297 (ObjNum 1628 snap 0 seq_num 1628) dirty exists 2014-07-01 16:53:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 1 to mira0435601-120 2014-07-01 16:53:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 2 to mira0435601-120 2014-07-01 16:53:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 3 to mira0435601-120 2014-07-01 16:53:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 4 to mira0435601-120 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 6 to mira0435601-120 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 7 to mira0435601-120 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 139 (ObjNum 1629 snap 0 seq_num 1629) dirty exists 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 1 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 2 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 3 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 4 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 6 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: finishing write tid 7 to mira0435601-418 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 271 (ObjNum 1630 snap 0 seq_num 1630) dirty exists 2014-07-01 16:53:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2549: expect (ObjNum 1590 snap 0 seq_num 1590) 2014-07-01 16:53:47,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:711: done (2 left) 2014-07-01 16:53:47,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:712: done (1 left) 2014-07-01 16:53:47,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:713: done (0 left) 2014-07-01 16:53:47,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:714: delete oid 96 current snap is 30 2014-07-01 16:53:47,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:714: done (0 left) 2014-07-01 16:53:47,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:715: rollback oid 259 current snap is 30 2014-07-01 16:53:47,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 259 to 18 2014-07-01 16:53:47,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: snap_create 2014-07-01 16:53:47,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 112 (ObjNum 258 snap 0 seq_num 258) dirty exists 2014-07-01 16:53:47,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:715: done (1 left) 2014-07-01 16:53:47,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:716: done (0 left) 2014-07-01 16:53:47,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: read oid 120 snap -1 2014-07-01 16:53:47,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: read oid 218 snap -1 2014-07-01 16:53:47,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: rollback oid 100 current snap is 31 2014-07-01 16:53:47,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 100 to 20 2014-07-01 16:53:47,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:720: snap_remove snap 27 2014-07-01 16:53:47,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 142 (ObjNum 99 snap 0 seq_num 99) dirty exists 2014-07-01 16:53:47,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: expect (ObjNum 217 snap 0 seq_num 217) 2014-07-01 16:53:48,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: expect (ObjNum 560 snap 26 seq_num 560) 2014-07-01 16:53:48,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: expect (ObjNum 642 snap 0 seq_num 642) 2014-07-01 16:53:48,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:717: done (3 left) 2014-07-01 16:53:48,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:718: done (2 left) 2014-07-01 16:53:48,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:719: done (1 left) 2014-07-01 16:53:48,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:720: done (0 left) 2014-07-01 16:53:48,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: snap_remove snap 18 2014-07-01 16:53:49,123.123 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:49,123.123 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:49,123.123 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_2 2014-07-01 16:53:49,124.124 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:53:49,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:721: done (0 left) 2014-07-01 16:53:49,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: write oid 146 current snap is 31 2014-07-01 16:53:49,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: seq_num 582 ranges {795870=1} 2014-07-01 16:53:49,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: writing mira0437250-146 from 795870 to 795871 tid 1 2014-07-01 16:53:49,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: rollback oid 115 current snap is 31 2014-07-01 16:53:49,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 115 to 9 2014-07-01 16:53:49,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:724: snap_create 2014-07-01 16:53:49,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: finishing write tid 1 to mira0437250-146 2014-07-01 16:53:49,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 147 (ObjNum 114 snap 0 seq_num 114) dirty exists 2014-07-01 16:53:49,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: finishing write tid 3 to mira0437250-146 2014-07-01 16:53:49,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: finishing write tid 4 to mira0437250-146 2014-07-01 16:53:49,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 178 (ObjNum 582 snap 31 seq_num 582) dirty exists 2014-07-01 16:53:49,744.744 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:53:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: finishing write tid 1 to mira0435601-253 2014-07-01 16:53:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: finishing write tid 2 to mira0435601-253 2014-07-01 16:53:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: finishing write tid 4 to mira0435601-253 2014-07-01 16:53:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: finishing write tid 5 to mira0435601-253 2014-07-01 16:53:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 346 (ObjNum 1631 snap 0 seq_num 1631) dirty exists 2014-07-01 16:53:49,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: seq_num 1632 ranges {872448=458752,1331200=434176,1765376=614400,2379776=155648} 2014-07-01 16:53:49,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: writing mira0435601-107 from 872448 to 1331200 tid 1 2014-07-01 16:53:49,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: writing mira0435601-107 from 1331200 to 1765376 tid 2 2014-07-01 16:53:49,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: writing mira0435601-107 from 1765376 to 2379776 tid 3 2014-07-01 16:53:49,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: writing mira0435601-107 from 2379776 to 2535424 tid 4 2014-07-01 16:53:49,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: done (6 left) 2014-07-01 16:53:49,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: done (5 left) 2014-07-01 16:53:49,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: done (4 left) 2014-07-01 16:53:49,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: done (3 left) 2014-07-01 16:53:49,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2549: done (2 left) 2014-07-01 16:53:49,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: done (1 left) 2014-07-01 16:53:49,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 495 current snap is 0 2014-07-01 16:53:49,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: seq_num 1633 ranges {3002368=729088,3731456=421888,4153344=380928} 2014-07-01 16:53:50,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: writing mira0435601-495 from 3002368 to 3731456 tid 1 2014-07-01 16:53:50,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: writing mira0435601-495 from 3731456 to 4153344 tid 2 2014-07-01 16:53:50,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: writing mira0435601-495 from 4153344 to 4534272 tid 3 2014-07-01 16:53:50,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: read oid 425 snap -1 2014-07-01 16:53:50,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 1 to mira0435601-107 2014-07-01 16:53:50,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 2 to mira0435601-107 2014-07-01 16:53:50,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 3 to mira0435601-107 2014-07-01 16:53:50,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 4 to mira0435601-107 2014-07-01 16:53:50,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 71 current snap is 0 2014-07-01 16:53:50,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: seq_num 1634 ranges {3141632=356352} 2014-07-01 16:53:50,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: writing mira0435601-71 from 3141632 to 3497984 tid 1 2014-07-01 16:53:50,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2555: read oid 79 snap -1 2014-07-01 16:53:50,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 460 current snap is 0 2014-07-01 16:53:50,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: seq_num 1635 ranges {2621440=229376} 2014-07-01 16:53:50,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: writing mira0435601-460 from 2621440 to 2850816 tid 1 2014-07-01 16:53:50,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 221 current snap is 0 2014-07-01 16:53:50,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: seq_num 1636 ranges {1716224=98304} 2014-07-01 16:53:50,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: writing mira0435601-221 from 1716224 to 1814528 tid 1 2014-07-01 16:53:50,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 16:53:50,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: seq_num 1637 ranges {2674688=577536,3252224=647168,3899392=733184,4632576=225280} 2014-07-01 16:53:50,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: writing mira0435601-176 from 2674688 to 3252224 tid 1 2014-07-01 16:53:50,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: writing mira0435601-176 from 3252224 to 3899392 tid 2 2014-07-01 16:53:50,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: writing mira0435601-176 from 3899392 to 4632576 tid 3 2014-07-01 16:53:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: writing mira0435601-176 from 4632576 to 4857856 tid 4 2014-07-01 16:53:50,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 140 current snap is 0 2014-07-01 16:53:50,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: seq_num 1638 ranges {6533120=606208,7139328=401408} 2014-07-01 16:53:50,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: writing mira0435601-140 from 6533120 to 7139328 tid 1 2014-07-01 16:53:50,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: writing mira0435601-140 from 7139328 to 7540736 tid 2 2014-07-01 16:53:50,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: read oid 14 snap -1 2014-07-01 16:53:50,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2561: read oid 298 snap -1 2014-07-01 16:53:50,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 6 to mira0435601-107 2014-07-01 16:53:50,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: finishing write tid 7 to mira0435601-107 2014-07-01 16:53:50,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 107 v 226 (ObjNum 1632 snap 0 seq_num 1632) dirty exists 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: finishing write tid 1 to mira0435601-495 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: finishing write tid 2 to mira0435601-495 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: finishing write tid 3 to mira0435601-495 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: finishing write tid 5 to mira0435601-495 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: done (10 left) 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: read oid 259 snap -1 2014-07-01 16:53:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: expect (ObjNum 1146 snap 0 seq_num 1146) 2014-07-01 16:53:50,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:722: done (2 left) 2014-07-01 16:53:50,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:723: done (1 left) 2014-07-01 16:53:50,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:724: done (0 left) 2014-07-01 16:53:50,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:725: delete oid 72 current snap is 32 2014-07-01 16:53:50,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:725: done (0 left) 2014-07-01 16:53:50,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:726: snap_create 2014-07-01 16:53:51,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:726: done (0 left) 2014-07-01 16:53:51,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:727: snap_create 2014-07-01 16:53:52,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: finishing write tid 6 to mira0435601-495 2014-07-01 16:53:52,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 204 (ObjNum 1633 snap 0 seq_num 1633) dirty exists 2014-07-01 16:53:52,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 1 to mira0435601-71 2014-07-01 16:53:52,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 1 to mira0435601-221 2014-07-01 16:53:52,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 3 to mira0435601-221 2014-07-01 16:53:52,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 4 to mira0435601-221 2014-07-01 16:53:52,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 348 (ObjNum 1636 snap 0 seq_num 1636) dirty exists 2014-07-01 16:53:52,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 3 to mira0435601-71 2014-07-01 16:53:52,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 1 to mira0435601-176 2014-07-01 16:53:52,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 2 to mira0435601-176 2014-07-01 16:53:52,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 3 to mira0435601-176 2014-07-01 16:53:52,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 4 to mira0435601-176 2014-07-01 16:53:52,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 6 to mira0435601-176 2014-07-01 16:53:52,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: finishing write tid 7 to mira0435601-176 2014-07-01 16:53:52,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 230 (ObjNum 1637 snap 0 seq_num 1637) dirty exists 2014-07-01 16:53:52,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2555: expect (ObjNum 1242 snap 0 seq_num 1242) 2014-07-01 16:53:52,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:727: done (0 left) 2014-07-01 16:53:52,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: read oid 183 snap -1 2014-07-01 16:53:52,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: delete oid 126 current snap is 34 2014-07-01 16:53:52,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:729: done (1 left) 2014-07-01 16:53:52,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: snap_create 2014-07-01 16:53:52,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 16:53:52,686.686 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:53:52,686.686 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pg_num 26' 2014-07-01 16:53:53,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:728: done (1 left) 2014-07-01 16:53:53,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:730: done (0 left) 2014-07-01 16:53:53,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: read oid 419 snap 33 2014-07-01 16:53:53,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: read oid 127 snap -1 2014-07-01 16:53:53,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: write oid 243 current snap is 35 2014-07-01 16:53:53,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: seq_num 585 ranges {779736=789156,1979735=1} 2014-07-01 16:53:53,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: writing mira0437250-243 from 779736 to 1568892 tid 1 2014-07-01 16:53:53,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: writing mira0437250-243 from 1979735 to 1979736 tid 2 2014-07-01 16:53:53,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:734: snap_create 2014-07-01 16:53:53,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: expect (ObjNum 418 snap 0 seq_num 418) 2014-07-01 16:53:53,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: expect (ObjNum 126 snap 0 seq_num 126) 2014-07-01 16:53:53,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 4 to mira0435601-71 2014-07-01 16:53:53,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 205 (ObjNum 1634 snap 0 seq_num 1634) dirty exists 2014-07-01 16:53:53,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 1 to mira0435601-460 2014-07-01 16:53:53,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 3 to mira0435601-460 2014-07-01 16:53:53,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 4 to mira0435601-460 2014-07-01 16:53:53,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 147 (ObjNum 1635 snap 0 seq_num 1635) dirty exists 2014-07-01 16:53:53,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 1 to mira0435601-140 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 2 to mira0435601-140 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: done (10 left) 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: done (9 left) 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: done (8 left) 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2555: done (7 left) 2014-07-01 16:53:53,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: done (6 left) 2014-07-01 16:53:53,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: done (5 left) 2014-07-01 16:53:53,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: done (4 left) 2014-07-01 16:53:53,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 484 current snap is 0 2014-07-01 16:53:53,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: seq_num 1639 ranges {1417216=720896,2138112=462848,2600960=434176,3035136=307200} 2014-07-01 16:53:53,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: finishing write tid 1 to mira0437250-243 2014-07-01 16:53:53,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: finishing write tid 2 to mira0437250-243 2014-07-01 16:53:53,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: finishing write tid 4 to mira0437250-243 2014-07-01 16:53:53,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: finishing write tid 5 to mira0437250-243 2014-07-01 16:53:53,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 124 (ObjNum 585 snap 35 seq_num 585) dirty exists 2014-07-01 16:53:53,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: writing mira0435601-484 from 1417216 to 2138112 tid 1 2014-07-01 16:53:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: writing mira0435601-484 from 2138112 to 2600960 tid 2 2014-07-01 16:53:53,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: writing mira0435601-484 from 2600960 to 3035136 tid 3 2014-07-01 16:53:53,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: writing mira0435601-484 from 3035136 to 3342336 tid 4 2014-07-01 16:53:53,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 402 current snap is 0 2014-07-01 16:53:53,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: seq_num 1640 ranges {835584=749568} 2014-07-01 16:53:53,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: writing mira0435601-402 from 835584 to 1585152 tid 1 2014-07-01 16:53:53,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 108 current snap is 0 2014-07-01 16:53:53,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 4 to mira0435601-140 2014-07-01 16:53:53,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: expect (ObjNum 1379 snap 0 seq_num 1379) 2014-07-01 16:53:54,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:731: done (3 left) 2014-07-01 16:53:54,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:732: done (2 left) 2014-07-01 16:53:54,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:733: done (1 left) 2014-07-01 16:53:54,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:734: done (0 left) 2014-07-01 16:53:54,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: rollback oid 226 current snap is 36 2014-07-01 16:53:54,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 226 to 32 2014-07-01 16:53:54,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: delete oid 210 current snap is 36 2014-07-01 16:53:54,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 179 (ObjNum 225 snap 0 seq_num 225) dirty exists 2014-07-01 16:53:54,396.396 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pg_num to 26 2014-07-01 16:53:55,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2561: expect (ObjNum 1302 snap 0 seq_num 1302) 2014-07-01 16:53:55,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:735: done (1 left) 2014-07-01 16:53:55,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:736: done (0 left) 2014-07-01 16:53:55,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: snap_create 2014-07-01 16:53:56,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:737: done (0 left) 2014-07-01 16:53:56,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: snap_remove snap 29 2014-07-01 16:53:57,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:738: done (0 left) 2014-07-01 16:53:57,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: write oid 355 current snap is 37 2014-07-01 16:53:57,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: seq_num 587 ranges {785747=768860,2095466=738700,3445772=539975} 2014-07-01 16:53:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: writing mira0437250-355 from 785747 to 1554607 tid 1 2014-07-01 16:53:57,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: writing mira0437250-355 from 2095466 to 2834166 tid 2 2014-07-01 16:53:57,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: writing mira0437250-355 from 3445772 to 3985747 tid 3 2014-07-01 16:53:57,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: read oid 95 snap -1 2014-07-01 16:53:57,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: finishing write tid 1 to mira0437250-355 2014-07-01 16:53:57,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: finishing write tid 2 to mira0437250-355 2014-07-01 16:53:57,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: snap_create 2014-07-01 16:53:57,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: finishing write tid 3 to mira0437250-355 2014-07-01 16:53:57,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: finishing write tid 5 to mira0437250-355 2014-07-01 16:53:57,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: finishing write tid 6 to mira0437250-355 2014-07-01 16:53:57,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 131 (ObjNum 587 snap 37 seq_num 587) dirty exists 2014-07-01 16:53:57,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: expect (ObjNum 94 snap 0 seq_num 94) 2014-07-01 16:53:57,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 5 to mira0435601-140 2014-07-01 16:53:57,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 150 (ObjNum 1638 snap 0 seq_num 1638) dirty exists 2014-07-01 16:53:57,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: expect (ObjNum 1569 snap 0 seq_num 1569) 2014-07-01 16:53:58,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:739: done (2 left) 2014-07-01 16:53:58,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:740: done (1 left) 2014-07-01 16:53:58,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:741: done (0 left) 2014-07-01 16:53:58,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:742: delete oid 312 current snap is 38 2014-07-01 16:53:58,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:742: done (0 left) 2014-07-01 16:53:58,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: read oid 152 snap -1 2014-07-01 16:53:58,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: write oid 488 current snap is 38 2014-07-01 16:53:58,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: seq_num 589 ranges {424782=1} 2014-07-01 16:53:58,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: writing mira0437250-488 from 424782 to 424783 tid 1 2014-07-01 16:53:58,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: rollback oid 425 current snap is 38 2014-07-01 16:53:58,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 425 to 20 2014-07-01 16:53:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: read oid 498 snap -1 2014-07-01 16:53:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: read oid 287 snap -1 2014-07-01 16:53:58,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:748: rollback oid 266 current snap is 38 2014-07-01 16:53:58,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 266 to 33 2014-07-01 16:53:58,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: write oid 352 current snap is 38 2014-07-01 16:53:58,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: seq_num 590 ranges {120012=1} 2014-07-01 16:53:58,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: writing mira0437250-352 from 120012 to 120013 tid 1 2014-07-01 16:53:58,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:750: snap_create 2014-07-01 16:53:58,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: expect (ObjNum 517 snap 10 seq_num 517) 2014-07-01 16:53:58,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 1 to mira0437250-488 2014-07-01 16:53:58,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 151 (ObjNum 424 snap 0 seq_num 424) dirty exists 2014-07-01 16:53:58,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 3 to mira0437250-488 2014-07-01 16:53:58,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: finishing write tid 1 to mira0437250-352 2014-07-01 16:53:58,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: finishing write tid 3 to mira0437250-352 2014-07-01 16:53:58,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 128 (ObjNum 265 snap 0 seq_num 265) dirty exists 2014-07-01 16:53:58,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: finishing write tid 4 to mira0437250-488 2014-07-01 16:53:58,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 134 (ObjNum 589 snap 38 seq_num 589) dirty exists 2014-07-01 16:53:58,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: finishing write tid 4 to mira0437250-352 2014-07-01 16:53:58,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 352 v 232 (ObjNum 590 snap 38 seq_num 590) dirty exists 2014-07-01 16:53:58,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: expect (ObjNum 151 snap 0 seq_num 151) 2014-07-01 16:53:58,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: expect (ObjNum 520 snap 11 seq_num 520) 2014-07-01 16:53:59,408.408 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:53:59,409.409 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:53:59,409.409 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 16:53:59,410.410 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:53:59,410.410 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:53:59,410.410 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 16:53:59,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:743: done (7 left) 2014-07-01 16:53:59,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:744: done (6 left) 2014-07-01 16:53:59,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:745: done (5 left) 2014-07-01 16:53:59,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:746: done (4 left) 2014-07-01 16:53:59,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:747: done (3 left) 2014-07-01 16:53:59,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:748: done (2 left) 2014-07-01 16:53:59,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:749: done (1 left) 2014-07-01 16:53:59,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:750: done (0 left) 2014-07-01 16:53:59,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: read oid 494 snap -1 2014-07-01 16:53:59,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: write oid 479 current snap is 39 2014-07-01 16:53:59,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: seq_num 591 ranges {581194=693198,1381193=1} 2014-07-01 16:53:59,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: writing mira0437250-479 from 581194 to 1274392 tid 1 2014-07-01 16:53:59,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: writing mira0437250-479 from 1381193 to 1381194 tid 2 2014-07-01 16:53:59,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: read oid 23 snap -1 2014-07-01 16:53:59,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: expect (ObjNum 493 snap 0 seq_num 493) 2014-07-01 16:54:00,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 1 to mira0437250-479 2014-07-01 16:54:00,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 2 to mira0437250-479 2014-07-01 16:54:00,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 4 to mira0437250-479 2014-07-01 16:54:00,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: finishing write tid 5 to mira0437250-479 2014-07-01 16:54:00,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 187 (ObjNum 591 snap 39 seq_num 591) dirty exists 2014-07-01 16:54:00,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: expect (ObjNum 22 snap 0 seq_num 22) 2014-07-01 16:54:00,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:751: done (2 left) 2014-07-01 16:54:00,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:752: done (1 left) 2014-07-01 16:54:00,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:753: done (0 left) 2014-07-01 16:54:00,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:754: snap_create 2014-07-01 16:54:01,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 1 to mira0435601-484 2014-07-01 16:54:01,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 2 to mira0435601-484 2014-07-01 16:54:01,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 3 to mira0435601-484 2014-07-01 16:54:01,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 4 to mira0435601-484 2014-07-01 16:54:01,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 6 to mira0435601-484 2014-07-01 16:54:01,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: finishing write tid 7 to mira0435601-484 2014-07-01 16:54:01,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 302 (ObjNum 1639 snap 0 seq_num 1639) dirty exists 2014-07-01 16:54:01,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: finishing write tid 1 to mira0435601-402 2014-07-01 16:54:01,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: finishing write tid 3 to mira0435601-402 2014-07-01 16:54:01,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: finishing write tid 4 to mira0435601-402 2014-07-01 16:54:01,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 273 (ObjNum 1640 snap 0 seq_num 1640) dirty exists 2014-07-01 16:54:01,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: seq_num 1641 ranges {0=421888} 2014-07-01 16:54:01,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: writing mira0435601-108 from 0 to 421888 tid 1 2014-07-01 16:54:01,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: done (6 left) 2014-07-01 16:54:01,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: done (5 left) 2014-07-01 16:54:01,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2561: done (4 left) 2014-07-01 16:54:01,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: done (3 left) 2014-07-01 16:54:01,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: done (2 left) 2014-07-01 16:54:01,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: done (1 left) 2014-07-01 16:54:01,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: read oid 380 snap -1 2014-07-01 16:54:01,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2567: read oid 138 snap -1 2014-07-01 16:54:01,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 193 current snap is 0 2014-07-01 16:54:01,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: seq_num 1642 ranges {5120000=540672,5660672=454656,6115328=348160} 2014-07-01 16:54:01,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:754: done (0 left) 2014-07-01 16:54:01,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:755: rollback oid 342 current snap is 40 2014-07-01 16:54:01,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 342 to 21 2014-07-01 16:54:01,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:756: delete oid 104 current snap is 40 2014-07-01 16:54:01,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 153 (ObjNum 341 snap 0 seq_num 341) dirty exists 2014-07-01 16:54:01,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:755: done (1 left) 2014-07-01 16:54:01,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:756: done (0 left) 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:757: rollback oid 426 current snap is 40 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 426 to 32 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: rollback oid 200 current snap is 40 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 200 to 32 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: read oid 54 snap -1 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: write oid 214 current snap is 40 2014-07-01 16:54:01,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: seq_num 593 ranges {621254=788230,1821253=1} 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: writing mira0437250-214 from 621254 to 1409484 tid 1 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: writing mira0437250-214 from 1821253 to 1821254 tid 2 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: write oid 57 current snap is 40 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: seq_num 594 ranges {566084=512483,1835704=483314,2566083=1} 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: writing mira0437250-57 from 566084 to 1078567 tid 1 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: writing mira0437250-57 from 1835704 to 2319018 tid 2 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: writing mira0437250-57 from 2566083 to 2566084 tid 3 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:762: snap_remove snap 36 2014-07-01 16:54:01,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 165 (ObjNum 199 snap 0 seq_num 199) dirty exists 2014-07-01 16:54:01,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 426 v 152 (ObjNum 425 snap 0 seq_num 425) dirty exists 2014-07-01 16:54:01,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: expect (ObjNum 53 snap 0 seq_num 53) 2014-07-01 16:54:01,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: writing mira0435601-193 from 5120000 to 5660672 tid 1 2014-07-01 16:54:01,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: writing mira0435601-193 from 5660672 to 6115328 tid 2 2014-07-01 16:54:01,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: writing mira0435601-193 from 6115328 to 6463488 tid 3 2014-07-01 16:54:01,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2569: read oid 412 snap -1 2014-07-01 16:54:01,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: finishing write tid 1 to mira0435601-108 2014-07-01 16:54:01,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: finishing write tid 3 to mira0435601-108 2014-07-01 16:54:01,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 16:54:02,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: finishing write tid 1 to mira0435601-193 2014-07-01 16:54:02,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: finishing write tid 2 to mira0435601-193 2014-07-01 16:54:02,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: finishing write tid 3 to mira0435601-193 2014-07-01 16:54:02,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: finishing write tid 5 to mira0435601-193 2014-07-01 16:54:02,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:2569: expect (ObjNum 1053 snap 0 seq_num 1053) 2014-07-01 16:54:02,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: finishing write tid 1 to mira0437250-214 2014-07-01 16:54:02,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: finishing write tid 2 to mira0437250-214 2014-07-01 16:54:02,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: finishing write tid 4 to mira0437250-214 2014-07-01 16:54:02,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: finishing write tid 5 to mira0437250-214 2014-07-01 16:54:02,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 187 (ObjNum 593 snap 40 seq_num 593) dirty exists 2014-07-01 16:54:02,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: finishing write tid 1 to mira0437250-57 2014-07-01 16:54:02,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: finishing write tid 2 to mira0437250-57 2014-07-01 16:54:02,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: finishing write tid 3 to mira0437250-57 2014-07-01 16:54:02,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: finishing write tid 5 to mira0437250-57 2014-07-01 16:54:02,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: finishing write tid 6 to mira0437250-57 2014-07-01 16:54:02,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 158 (ObjNum 594 snap 40 seq_num 594) dirty exists 2014-07-01 16:54:02,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:757: done (5 left) 2014-07-01 16:54:02,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:758: done (4 left) 2014-07-01 16:54:02,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:759: done (3 left) 2014-07-01 16:54:02,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:760: done (2 left) 2014-07-01 16:54:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:761: done (1 left) 2014-07-01 16:54:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:762: done (0 left) 2014-07-01 16:54:02,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: write oid 166 current snap is 40 2014-07-01 16:54:02,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: seq_num 595 ranges {509889=767935,1681035=685733,3016151=293738} 2014-07-01 16:54:02,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0437250-166 from 509889 to 1277824 tid 1 2014-07-01 16:54:02,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0437250-166 from 1681035 to 2366768 tid 2 2014-07-01 16:54:02,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: writing mira0437250-166 from 3016151 to 3309889 tid 3 2014-07-01 16:54:02,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:764: snap_create 2014-07-01 16:54:02,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 1 to mira0437250-166 2014-07-01 16:54:02,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 2 to mira0437250-166 2014-07-01 16:54:02,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 3 to mira0437250-166 2014-07-01 16:54:02,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 5 to mira0437250-166 2014-07-01 16:54:02,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: finishing write tid 6 to mira0437250-166 2014-07-01 16:54:02,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 232 (ObjNum 595 snap 40 seq_num 595) dirty exists 2014-07-01 16:54:03,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: finishing write tid 6 to mira0435601-193 2014-07-01 16:54:03,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 306 (ObjNum 1642 snap 0 seq_num 1642) dirty exists 2014-07-01 16:54:03,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: done (4 left) 2014-07-01 16:54:03,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2567: done (3 left) 2014-07-01 16:54:03,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: done (2 left) 2014-07-01 16:54:03,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2569: done (1 left) 2014-07-01 16:54:03,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 103 current snap is 0 2014-07-01 16:54:03,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: seq_num 1643 ranges {1994752=167936} 2014-07-01 16:54:03,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: writing mira0435601-103 from 1994752 to 2162688 tid 1 2014-07-01 16:54:03,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2571: read oid 150 snap -1 2014-07-01 16:54:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 16:54:03,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: seq_num 1644 ranges {2207744=593920,2801664=671744,3473408=131072} 2014-07-01 16:54:03,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:763: done (1 left) 2014-07-01 16:54:03,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:764: done (0 left) 2014-07-01 16:54:03,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:765: snap_remove snap 11 2014-07-01 16:54:03,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: writing mira0435601-31 from 2207744 to 2801664 tid 1 2014-07-01 16:54:03,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: writing mira0435601-31 from 2801664 to 3473408 tid 2 2014-07-01 16:54:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: writing mira0435601-31 from 3473408 to 3604480 tid 3 2014-07-01 16:54:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2573: read oid 5 snap -1 2014-07-01 16:54:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 1 to mira0435601-103 2014-07-01 16:54:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 3 to mira0435601-103 2014-07-01 16:54:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: finishing write tid 1 to mira0435601-31 2014-07-01 16:54:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2571: done (4 left) 2014-07-01 16:54:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: read oid 2 snap -1 2014-07-01 16:54:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2575: delete oid 168 current snap is 0 2014-07-01 16:54:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: finishing write tid 2 to mira0435601-31 2014-07-01 16:54:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: finishing write tid 3 to mira0435601-31 2014-07-01 16:54:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: finishing write tid 5 to mira0435601-31 2014-07-01 16:54:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2573: done (5 left) 2014-07-01 16:54:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2575: done (4 left) 2014-07-01 16:54:03,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2576: read oid 238 snap -1 2014-07-01 16:54:03,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 217 current snap is 0 2014-07-01 16:54:03,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: seq_num 1646 ranges {3629056=790528,4419584=438272,4857856=450560,5308416=315392} 2014-07-01 16:54:03,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: writing mira0435601-217 from 3629056 to 4419584 tid 1 2014-07-01 16:54:03,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: writing mira0435601-217 from 4419584 to 4857856 tid 2 2014-07-01 16:54:03,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: writing mira0435601-217 from 4857856 to 5308416 tid 3 2014-07-01 16:54:03,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: writing mira0435601-217 from 5308416 to 5623808 tid 4 2014-07-01 16:54:03,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 379 current snap is 0 2014-07-01 16:54:03,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: seq_num 1647 ranges {0=659456,659456=421888,1081344=729088,1810432=53248} 2014-07-01 16:54:03,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: writing mira0435601-379 from 0 to 659456 tid 1 2014-07-01 16:54:03,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: writing mira0435601-379 from 659456 to 1081344 tid 2 2014-07-01 16:54:03,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: writing mira0435601-379 from 1081344 to 1810432 tid 3 2014-07-01 16:54:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: writing mira0435601-379 from 1810432 to 1863680 tid 4 2014-07-01 16:54:03,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 152 current snap is 0 2014-07-01 16:54:03,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: seq_num 1648 ranges {0=532480,532480=495616,1028096=552960,1581056=155648} 2014-07-01 16:54:03,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: writing mira0435601-152 from 0 to 532480 tid 1 2014-07-01 16:54:03,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: writing mira0435601-152 from 532480 to 1028096 tid 2 2014-07-01 16:54:03,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: writing mira0435601-152 from 1028096 to 1581056 tid 3 2014-07-01 16:54:03,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: writing mira0435601-152 from 1581056 to 1736704 tid 4 2014-07-01 16:54:03,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:2580: read oid 406 snap -1 2014-07-01 16:54:03,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: finishing write tid 6 to mira0435601-31 2014-07-01 16:54:03,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 264 (ObjNum 1644 snap 0 seq_num 1644) dirty exists 2014-07-01 16:54:03,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: expect (ObjNum 1546 snap 0 seq_num 1546) 2014-07-01 16:54:03,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 1 to mira0435601-217 2014-07-01 16:54:03,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 2 to mira0435601-217 2014-07-01 16:54:03,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 3 to mira0435601-217 2014-07-01 16:54:03,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 4 to mira0435601-217 2014-07-01 16:54:03,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 6 to mira0435601-217 2014-07-01 16:54:03,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 1 to mira0435601-379 2014-07-01 16:54:03,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 2 to mira0435601-379 2014-07-01 16:54:03,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 3 to mira0435601-379 2014-07-01 16:54:03,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 4 to mira0435601-379 2014-07-01 16:54:03,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 6 to mira0435601-379 2014-07-01 16:54:03,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 1 to mira0435601-152 2014-07-01 16:54:03,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 2 to mira0435601-152 2014-07-01 16:54:03,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 3 to mira0435601-152 2014-07-01 16:54:03,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 4 to mira0435601-152 2014-07-01 16:54:03,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 6 to mira0435601-152 2014-07-01 16:54:03,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2580: expect (ObjNum 1402 snap 0 seq_num 1402) 2014-07-01 16:54:04,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: done (8 left) 2014-07-01 16:54:04,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: done (7 left) 2014-07-01 16:54:04,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2576: done (6 left) 2014-07-01 16:54:04,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2580: done (5 left) 2014-07-01 16:54:04,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: read oid 94 snap -1 2014-07-01 16:54:04,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2582: read oid 128 snap -1 2014-07-01 16:54:04,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: read oid 373 snap -1 2014-07-01 16:54:04,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2584: read oid 459 snap -1 2014-07-01 16:54:04,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: read oid 233 snap -1 2014-07-01 16:54:04,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 104 current snap is 0 2014-07-01 16:54:04,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: seq_num 1649 ranges {0=561152} 2014-07-01 16:54:04,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: writing mira0435601-104 from 0 to 561152 tid 1 2014-07-01 16:54:04,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 259 current snap is 0 2014-07-01 16:54:04,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: seq_num 1650 ranges {1839104=479232,2318336=663552,2981888=139264} 2014-07-01 16:54:04,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: writing mira0435601-259 from 1839104 to 2318336 tid 1 2014-07-01 16:54:04,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: writing mira0435601-259 from 2318336 to 2981888 tid 2 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: writing mira0435601-259 from 2981888 to 3121152 tid 3 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: seq_num 1651 ranges {0=106496} 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: writing mira0435601-250 from 0 to 106496 tid 1 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: read oid 473 snap -1 2014-07-01 16:54:04,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: expect (ObjNum 1175 snap 0 seq_num 1175) 2014-07-01 16:54:04,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:765: done (0 left) 2014-07-01 16:54:04,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: read oid 118 snap -1 2014-07-01 16:54:04,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: write oid 192 current snap is 41 2014-07-01 16:54:04,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: seq_num 596 ranges {722495=491163,1662943=507352,2685152=549802,3522494=1} 2014-07-01 16:54:04,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0437250-192 from 722495 to 1213658 tid 1 2014-07-01 16:54:04,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0437250-192 from 1662943 to 2170295 tid 2 2014-07-01 16:54:04,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0437250-192 from 2685152 to 3234954 tid 3 2014-07-01 16:54:04,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: writing mira0437250-192 from 3522494 to 3522495 tid 4 2014-07-01 16:54:04,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: read oid 156 snap -1 2014-07-01 16:54:04,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: expect (ObjNum 514 snap 8 seq_num 514) 2014-07-01 16:54:04,886.886 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:04,886.886 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:04,886.886 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_2 2014-07-01 16:54:04,887.887 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:54:05,646.646 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:54:05,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: expect (ObjNum 698 snap 0 seq_num 698) 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 1 to mira0437250-192 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 2 to mira0437250-192 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 3 to mira0437250-192 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 4 to mira0437250-192 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 6 to mira0437250-192 2014-07-01 16:54:06,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: expect (ObjNum 155 snap 0 seq_num 155) 2014-07-01 16:54:07,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: finishing write tid 7 to mira0437250-192 2014-07-01 16:54:07,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 227 (ObjNum 596 snap 41 seq_num 596) dirty exists 2014-07-01 16:54:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:766: done (2 left) 2014-07-01 16:54:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:767: done (1 left) 2014-07-01 16:54:07,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:768: done (0 left) 2014-07-01 16:54:07,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: snap_create 2014-07-01 16:54:07,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:769: done (0 left) 2014-07-01 16:54:07,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:770: snap_remove snap 19 2014-07-01 16:54:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: finishing write tid 1 to mira0435601-259 2014-07-01 16:54:08,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: finishing write tid 2 to mira0435601-259 2014-07-01 16:54:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: finishing write tid 3 to mira0435601-259 2014-07-01 16:54:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: finishing write tid 5 to mira0435601-259 2014-07-01 16:54:08,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: done (13 left) 2014-07-01 16:54:08,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: done (12 left) 2014-07-01 16:54:08,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2590: read oid 172 snap -1 2014-07-01 16:54:08,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: expect (ObjNum 1377 snap 0 seq_num 1377) 2014-07-01 16:54:08,556.556 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:54:08,556.556 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pg_num 36' 2014-07-01 16:54:08,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:770: done (0 left) 2014-07-01 16:54:08,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: delete oid 307 current snap is 42 2014-07-01 16:54:08,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:771: done (0 left) 2014-07-01 16:54:08,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: write oid 366 current snap is 42 2014-07-01 16:54:08,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: seq_num 598 ranges {610106=400000} 2014-07-01 16:54:08,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: writing mira0437250-366 from 610106 to 1010106 tid 1 2014-07-01 16:54:08,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:773: delete oid 405 current snap is 42 2014-07-01 16:54:08,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:773: done (1 left) 2014-07-01 16:54:08,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: write oid 414 current snap is 42 2014-07-01 16:54:08,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: seq_num 600 ranges {517942=400000} 2014-07-01 16:54:08,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: writing mira0437250-414 from 517942 to 917942 tid 1 2014-07-01 16:54:08,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: delete oid 148 current snap is 42 2014-07-01 16:54:08,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: finishing write tid 1 to mira0437250-366 2014-07-01 16:54:08,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: finishing write tid 3 to mira0437250-366 2014-07-01 16:54:08,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: finishing write tid 4 to mira0437250-366 2014-07-01 16:54:08,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 214 (ObjNum 598 snap 42 seq_num 598) dirty exists 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:772: done (2 left) 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:775: done (1 left) 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:776: snap_create 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: finishing write tid 1 to mira0437250-414 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: finishing write tid 3 to mira0437250-414 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: finishing write tid 4 to mira0437250-414 2014-07-01 16:54:08,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 270 (ObjNum 600 snap 42 seq_num 600) dirty exists 2014-07-01 16:54:09,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:774: done (1 left) 2014-07-01 16:54:09,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:776: done (0 left) 2014-07-01 16:54:09,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: read oid 440 snap -1 2014-07-01 16:54:09,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: snap_create 2014-07-01 16:54:09,547.547 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pg_num to 36 2014-07-01 16:54:10,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:778: done (1 left) 2014-07-01 16:54:10,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:779: snap_remove snap 9 2014-07-01 16:54:10,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: expect (ObjNum 439 snap 0 seq_num 439) 2014-07-01 16:54:11,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:777: done (1 left) 2014-07-01 16:54:11,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:779: done (0 left) 2014-07-01 16:54:11,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: write oid 84 current snap is 44 2014-07-01 16:54:11,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: seq_num 602 ranges {407601=510533,1689048=431372,2407600=1} 2014-07-01 16:54:11,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: writing mira0437250-84 from 407601 to 918134 tid 1 2014-07-01 16:54:11,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: writing mira0437250-84 from 1689048 to 2120420 tid 2 2014-07-01 16:54:11,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: writing mira0437250-84 from 2407600 to 2407601 tid 3 2014-07-01 16:54:11,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: read oid 179 snap -1 2014-07-01 16:54:11,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: write oid 74 current snap is 44 2014-07-01 16:54:11,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: seq_num 603 ranges {502831=765986,1685805=417026} 2014-07-01 16:54:11,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: writing mira0437250-74 from 502831 to 1268817 tid 1 2014-07-01 16:54:11,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: writing mira0437250-74 from 1685805 to 2102831 tid 2 2014-07-01 16:54:11,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:783: snap_remove snap 30 2014-07-01 16:54:11,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: finishing write tid 1 to mira0437250-84 2014-07-01 16:54:11,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: expect (ObjNum 178 snap 0 seq_num 178) 2014-07-01 16:54:13,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: finishing write tid 2 to mira0437250-84 2014-07-01 16:54:13,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: finishing write tid 3 to mira0437250-84 2014-07-01 16:54:13,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: finishing write tid 5 to mira0437250-84 2014-07-01 16:54:13,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: finishing write tid 6 to mira0437250-84 2014-07-01 16:54:13,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 168 (ObjNum 602 snap 44 seq_num 602) dirty exists 2014-07-01 16:54:13,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: finishing write tid 1 to mira0437250-74 2014-07-01 16:54:13,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: finishing write tid 2 to mira0437250-74 2014-07-01 16:54:13,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: finishing write tid 4 to mira0437250-74 2014-07-01 16:54:13,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: finishing write tid 5 to mira0437250-74 2014-07-01 16:54:13,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 178 (ObjNum 603 snap 44 seq_num 603) dirty exists 2014-07-01 16:54:13,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:780: done (3 left) 2014-07-01 16:54:13,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:781: done (2 left) 2014-07-01 16:54:13,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:782: done (1 left) 2014-07-01 16:54:13,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:783: done (0 left) 2014-07-01 16:54:13,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:784: rollback oid 311 current snap is 44 2014-07-01 16:54:13,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 311 to 38 2014-07-01 16:54:13,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: delete oid 89 current snap is 44 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:785: done (1 left) 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: write oid 2 current snap is 44 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: seq_num 605 ranges {107818=1} 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: writing mira0437250-2 from 107818 to 107819 tid 1 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: read oid 14 snap -1 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: snap_remove snap 43 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 302 (ObjNum 310 snap 0 seq_num 310) dirty exists 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 1 to mira0437250-2 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 3 to mira0437250-2 2014-07-01 16:54:13,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: expect (ObjNum 13 snap 0 seq_num 13) 2014-07-01 16:54:13,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: finishing write tid 4 to mira0437250-2 2014-07-01 16:54:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 2 v 155 (ObjNum 605 snap 44 seq_num 605) dirty exists 2014-07-01 16:54:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:784: done (3 left) 2014-07-01 16:54:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:786: done (2 left) 2014-07-01 16:54:13,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:787: done (1 left) 2014-07-01 16:54:13,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:788: done (0 left) 2014-07-01 16:54:13,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: write oid 59 current snap is 44 2014-07-01 16:54:13,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: seq_num 606 ranges {603048=585271,1707433=747224,2889662=783595,3803047=1} 2014-07-01 16:54:13,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0437250-59 from 603048 to 1188319 tid 1 2014-07-01 16:54:13,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0437250-59 from 1707433 to 2454657 tid 2 2014-07-01 16:54:13,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0437250-59 from 2889662 to 3673257 tid 3 2014-07-01 16:54:13,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: writing mira0437250-59 from 3803047 to 3803048 tid 4 2014-07-01 16:54:13,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 1 to mira0437250-59 2014-07-01 16:54:13,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: read oid 60 snap 31 2014-07-01 16:54:13,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 2 to mira0437250-59 2014-07-01 16:54:13,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: write oid 291 current snap is 44 2014-07-01 16:54:13,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: seq_num 607 ranges {796454=440723,1596453=1} 2014-07-01 16:54:13,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: writing mira0437250-291 from 796454 to 1237177 tid 1 2014-07-01 16:54:13,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: writing mira0437250-291 from 1596453 to 1596454 tid 2 2014-07-01 16:54:13,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:792: read oid 272 snap -1 2014-07-01 16:54:13,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 3 to mira0437250-59 2014-07-01 16:54:13,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 4 to mira0437250-59 2014-07-01 16:54:13,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 6 to mira0437250-59 2014-07-01 16:54:13,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: finishing write tid 7 to mira0437250-59 2014-07-01 16:54:13,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 178 (ObjNum 606 snap 44 seq_num 606) dirty exists 2014-07-01 16:54:13,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: expect (ObjNum 59 snap 0 seq_num 59) 2014-07-01 16:54:13,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: finishing write tid 1 to mira0435601-250 2014-07-01 16:54:13,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: finishing write tid 3 to mira0435601-250 2014-07-01 16:54:13,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: expect (ObjNum 731 snap 0 seq_num 731) 2014-07-01 16:54:14,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:792: expect (ObjNum 271 snap 0 seq_num 271) 2014-07-01 16:54:14,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: finishing write tid 1 to mira0437250-291 2014-07-01 16:54:14,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: finishing write tid 2 to mira0437250-291 2014-07-01 16:54:14,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: finishing write tid 4 to mira0437250-291 2014-07-01 16:54:14,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: finishing write tid 5 to mira0437250-291 2014-07-01 16:54:14,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 145 (ObjNum 607 snap 44 seq_num 607) dirty exists 2014-07-01 16:54:14,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:789: done (3 left) 2014-07-01 16:54:14,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:790: done (2 left) 2014-07-01 16:54:14,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:791: done (1 left) 2014-07-01 16:54:14,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:792: done (0 left) 2014-07-01 16:54:14,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:793: snap_remove snap 38 2014-07-01 16:54:14,577.577 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:14,577.577 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:14,578.578 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:54:14,578.578 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:54:14,578.578 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:54:14,579.579 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:54:14,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:793: done (0 left) 2014-07-01 16:54:14,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: snap_create 2014-07-01 16:54:14,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2584: expect (ObjNum 1364 snap 0 seq_num 1364) 2014-07-01 16:54:16,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:794: done (0 left) 2014-07-01 16:54:16,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:795: rollback oid 212 current snap is 45 2014-07-01 16:54:16,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 212 to 39 2014-07-01 16:54:16,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: snap_create 2014-07-01 16:54:16,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 202 (ObjNum 211 snap 0 seq_num 211) dirty exists 2014-07-01 16:54:16,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:795: done (1 left) 2014-07-01 16:54:16,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:796: done (0 left) 2014-07-01 16:54:16,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: snap_create 2014-07-01 16:54:17,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:797: done (0 left) 2014-07-01 16:54:17,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: write oid 166 current snap is 47 2014-07-01 16:54:17,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: seq_num 608 ranges {491684=564976,1291683=1} 2014-07-01 16:54:17,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: writing mira0437250-166 from 491684 to 1056660 tid 1 2014-07-01 16:54:17,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: writing mira0437250-166 from 1291683 to 1291684 tid 2 2014-07-01 16:54:17,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: delete oid 202 current snap is 47 2014-07-01 16:54:17,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:799: done (1 left) 2014-07-01 16:54:17,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: write oid 160 current snap is 47 2014-07-01 16:54:17,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: seq_num 610 ranges {497695=508584,1488121=530853,2539492=673413,3297694=1} 2014-07-01 16:54:17,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: writing mira0437250-160 from 497695 to 1006279 tid 1 2014-07-01 16:54:17,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: writing mira0437250-160 from 1488121 to 2018974 tid 2 2014-07-01 16:54:17,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: writing mira0437250-160 from 2539492 to 3212905 tid 3 2014-07-01 16:54:17,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: writing mira0437250-160 from 3297694 to 3297695 tid 4 2014-07-01 16:54:17,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:801: rollback oid 66 current snap is 47 2014-07-01 16:54:17,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 66 to 39 2014-07-01 16:54:17,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 1 to mira0437250-166 2014-07-01 16:54:17,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 2 to mira0437250-166 2014-07-01 16:54:17,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 4 to mira0437250-166 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: finishing write tid 5 to mira0437250-166 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 262 (ObjNum 608 snap 47 seq_num 608) dirty exists 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 1 to mira0437250-160 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 2 to mira0437250-160 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:798: done (2 left) 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: write oid 47 current snap is 47 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: seq_num 611 ranges {592925=765061,1885902=601048,2992924=1} 2014-07-01 16:54:17,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: writing mira0437250-47 from 592925 to 1357986 tid 1 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: writing mira0437250-47 from 1885902 to 2486950 tid 2 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: writing mira0437250-47 from 2992924 to 2992925 tid 3 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:803: snap_remove snap 40 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 204 (ObjNum 65 snap 0 seq_num 65) dirty exists 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 3 to mira0437250-160 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 4 to mira0437250-160 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 6 to mira0437250-160 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: finishing write tid 7 to mira0437250-160 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 206 (ObjNum 610 snap 47 seq_num 610) dirty exists 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: finishing write tid 1 to mira0437250-47 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: finishing write tid 2 to mira0437250-47 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: finishing write tid 3 to mira0437250-47 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: finishing write tid 5 to mira0437250-47 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: finishing write tid 6 to mira0437250-47 2014-07-01 16:54:17,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 256 (ObjNum 611 snap 47 seq_num 611) dirty exists 2014-07-01 16:54:18,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:800: done (3 left) 2014-07-01 16:54:18,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:801: done (2 left) 2014-07-01 16:54:18,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:802: done (1 left) 2014-07-01 16:54:18,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:803: done (0 left) 2014-07-01 16:54:18,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: delete oid 410 current snap is 47 2014-07-01 16:54:18,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:804: done (0 left) 2014-07-01 16:54:18,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:805: delete oid 222 current snap is 47 2014-07-01 16:54:18,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:805: done (0 left) 2014-07-01 16:54:18,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: read oid 237 snap -1 2014-07-01 16:54:18,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: write oid 244 current snap is 47 2014-07-01 16:54:18,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: seq_num 614 ranges {693141=1} 2014-07-01 16:54:18,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: writing mira0437250-244 from 693141 to 693142 tid 1 2014-07-01 16:54:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: write oid 286 current snap is 47 2014-07-01 16:54:18,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: seq_num 615 ranges {389395=1} 2014-07-01 16:54:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: writing mira0437250-286 from 389395 to 389396 tid 1 2014-07-01 16:54:18,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: snap_remove snap 33 2014-07-01 16:54:18,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: finishing write tid 1 to mira0437250-286 2014-07-01 16:54:18,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: finishing write tid 3 to mira0437250-286 2014-07-01 16:54:18,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: finishing write tid 4 to mira0437250-286 2014-07-01 16:54:18,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 255 (ObjNum 615 snap 47 seq_num 615) dirty exists 2014-07-01 16:54:19,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: finishing write tid 1 to mira0435601-104 2014-07-01 16:54:19,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: finishing write tid 3 to mira0435601-104 2014-07-01 16:54:19,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: finishing write tid 4 to mira0435601-104 2014-07-01 16:54:19,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 104 v 308 (ObjNum 1649 snap 0 seq_num 1649) dirty exists 2014-07-01 16:54:19,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: finishing write tid 4 to mira0435601-108 2014-07-01 16:54:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 232 (ObjNum 1641 snap 0 seq_num 1641) dirty exists 2014-07-01 16:54:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 4 to mira0435601-103 2014-07-01 16:54:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 228 (ObjNum 1643 snap 0 seq_num 1643) dirty exists 2014-07-01 16:54:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:2582: expect (ObjNum 1354 snap 0 seq_num 1354) 2014-07-01 16:54:19,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: finishing write tid 1 to mira0437250-244 2014-07-01 16:54:19,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: finishing write tid 3 to mira0437250-244 2014-07-01 16:54:19,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: finishing write tid 4 to mira0437250-244 2014-07-01 16:54:19,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 227 (ObjNum 614 snap 47 seq_num 614) dirty exists 2014-07-01 16:54:19,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:806: done (3 left) 2014-07-01 16:54:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:807: done (2 left) 2014-07-01 16:54:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:808: done (1 left) 2014-07-01 16:54:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:809: done (0 left) 2014-07-01 16:54:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: read oid 401 snap -1 2014-07-01 16:54:19,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:811: snap_create 2014-07-01 16:54:19,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: expect (ObjNum 400 snap 0 seq_num 400) 2014-07-01 16:54:20,059.059 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:20,060.060 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:54:20,649.649 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:54:20,661.661 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:54:21,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:810: done (1 left) 2014-07-01 16:54:21,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:811: done (0 left) 2014-07-01 16:54:21,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: snap_create 2014-07-01 16:54:21,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:812: done (0 left) 2014-07-01 16:54:21,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:813: snap_create 2014-07-01 16:54:21,718.718 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:54:21,734.734 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:54:22,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:813: done (0 left) 2014-07-01 16:54:22,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: write oid 463 current snap is 50 2014-07-01 16:54:22,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: seq_num 616 ranges {733202=400000} 2014-07-01 16:54:22,687.687 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:54:22,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: writing mira0437250-463 from 733202 to 1133202 tid 1 2014-07-01 16:54:22,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: write oid 476 current snap is 50 2014-07-01 16:54:22,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: seq_num 617 ranges {678031=534831,1662406=215625} 2014-07-01 16:54:22,700.700 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:54:22,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: writing mira0437250-476 from 678031 to 1212862 tid 1 2014-07-01 16:54:22,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: writing mira0437250-476 from 1662406 to 1878031 tid 2 2014-07-01 16:54:22,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: read oid 291 snap -1 2014-07-01 16:54:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: finishing write tid 1 to mira0437250-463 2014-07-01 16:54:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: finishing write tid 3 to mira0437250-463 2014-07-01 16:54:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: finishing write tid 4 to mira0437250-463 2014-07-01 16:54:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 269 (ObjNum 616 snap 50 seq_num 616) dirty exists 2014-07-01 16:54:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: finishing write tid 1 to mira0437250-476 2014-07-01 16:54:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:814: done (2 left) 2014-07-01 16:54:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: snap_create 2014-07-01 16:54:22,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: finishing write tid 2 to mira0437250-476 2014-07-01 16:54:22,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: finishing write tid 4 to mira0437250-476 2014-07-01 16:54:22,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: finishing write tid 5 to mira0437250-476 2014-07-01 16:54:22,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 273 (ObjNum 617 snap 50 seq_num 617) dirty exists 2014-07-01 16:54:22,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: expect (ObjNum 607 snap 44 seq_num 607) 2014-07-01 16:54:23,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:815: done (2 left) 2014-07-01 16:54:23,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:816: done (1 left) 2014-07-01 16:54:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:817: done (0 left) 2014-07-01 16:54:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:818: delete oid 415 current snap is 51 2014-07-01 16:54:23,705.705 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:54:23,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:818: done (0 left) 2014-07-01 16:54:23,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:819: delete oid 324 current snap is 51 2014-07-01 16:54:23,716.716 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:54:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:819: done (0 left) 2014-07-01 16:54:23,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:820: snap_create 2014-07-01 16:54:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:820: done (0 left) 2014-07-01 16:54:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: write oid 86 current snap is 52 2014-07-01 16:54:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: seq_num 620 ranges {779273=638788,1970756=769701,3333571=245702} 2014-07-01 16:54:24,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: writing mira0437250-86 from 779273 to 1418061 tid 1 2014-07-01 16:54:24,724.724 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:54:24,736.736 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:54:24,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: writing mira0437250-86 from 1970756 to 2740457 tid 2 2014-07-01 16:54:24,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: writing mira0437250-86 from 3333571 to 3579273 tid 3 2014-07-01 16:54:24,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:822: delete oid 29 current snap is 52 2014-07-01 16:54:24,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: finishing write tid 1 to mira0437250-86 2014-07-01 16:54:24,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: finishing write tid 2 to mira0437250-86 2014-07-01 16:54:24,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: finishing write tid 3 to mira0437250-86 2014-07-01 16:54:24,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: finishing write tid 5 to mira0437250-86 2014-07-01 16:54:24,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: finishing write tid 6 to mira0437250-86 2014-07-01 16:54:24,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 155 (ObjNum 620 snap 52 seq_num 620) dirty exists 2014-07-01 16:54:24,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:821: done (1 left) 2014-07-01 16:54:24,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:822: done (0 left) 2014-07-01 16:54:24,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: delete oid 53 current snap is 52 2014-07-01 16:54:24,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:823: done (0 left) 2014-07-01 16:54:24,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:824: snap_remove snap 21 2014-07-01 16:54:25,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:824: done (0 left) 2014-07-01 16:54:25,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:825: delete oid 336 current snap is 52 2014-07-01 16:54:25,736.736 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:54:25,749.749 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:54:25,750.750 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:54:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:825: done (0 left) 2014-07-01 16:54:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: read oid 186 snap -1 2014-07-01 16:54:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: read oid 199 snap 47 2014-07-01 16:54:25,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: write oid 366 current snap is 52 2014-07-01 16:54:25,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: seq_num 624 ranges {574720=400000} 2014-07-01 16:54:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: writing mira0437250-366 from 574720 to 974720 tid 1 2014-07-01 16:54:25,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: expect (ObjNum 198 snap 0 seq_num 198) 2014-07-01 16:54:25,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: read oid 275 snap 42 2014-07-01 16:54:25,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:827: done (3 left) 2014-07-01 16:54:25,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: rollback oid 181 current snap is 52 2014-07-01 16:54:25,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 181 to 46 2014-07-01 16:54:25,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: write oid 388 current snap is 52 2014-07-01 16:54:25,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: seq_num 625 ranges {519549=533905,1719548=1} 2014-07-01 16:54:25,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: writing mira0437250-388 from 519549 to 1053454 tid 1 2014-07-01 16:54:25,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: writing mira0437250-388 from 1719548 to 1719549 tid 2 2014-07-01 16:54:25,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: read oid 271 snap 34 2014-07-01 16:54:25,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 1 to mira0437250-366 2014-07-01 16:54:25,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 3 to mira0437250-366 2014-07-01 16:54:25,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 238 (ObjNum 180 snap 0 seq_num 180) dirty exists 2014-07-01 16:54:25,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:830: done (5 left) 2014-07-01 16:54:25,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: read oid 98 snap -1 2014-07-01 16:54:25,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:834: snap_create 2014-07-01 16:54:25,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: finishing write tid 4 to mira0437250-366 2014-07-01 16:54:25,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 264 (ObjNum 624 snap 52 seq_num 624) dirty exists 2014-07-01 16:54:25,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: expect (ObjNum 505 snap 3 seq_num 505) 2014-07-01 16:54:26,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: finishing write tid 7 to mira0435601-217 2014-07-01 16:54:26,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 221 (ObjNum 1646 snap 0 seq_num 1646) dirty exists 2014-07-01 16:54:26,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: finishing write tid 6 to mira0435601-259 2014-07-01 16:54:26,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 268 (ObjNum 1650 snap 0 seq_num 1650) dirty exists 2014-07-01 16:54:26,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: finishing write tid 7 to mira0435601-379 2014-07-01 16:54:26,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 237 (ObjNum 1647 snap 0 seq_num 1647) dirty exists 2014-07-01 16:54:26,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: finishing write tid 4 to mira0435601-250 2014-07-01 16:54:26,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 147 (ObjNum 1651 snap 0 seq_num 1651) dirty exists 2014-07-01 16:54:26,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: finishing write tid 7 to mira0435601-152 2014-07-01 16:54:26,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 145 (ObjNum 1648 snap 0 seq_num 1648) dirty exists 2014-07-01 16:54:26,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2590: expect (ObjNum 1609 snap 0 seq_num 1609) 2014-07-01 16:54:26,483.483 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:54:26,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: finishing write tid 1 to mira0437250-388 2014-07-01 16:54:26,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: finishing write tid 2 to mira0437250-388 2014-07-01 16:54:26,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: finishing write tid 4 to mira0437250-388 2014-07-01 16:54:26,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: finishing write tid 5 to mira0437250-388 2014-07-01 16:54:26,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 215 (ObjNum 625 snap 52 seq_num 625) dirty exists 2014-07-01 16:54:26,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: expect (ObjNum 185 snap 0 seq_num 185) 2014-07-01 16:54:27,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: expect (ObjNum 270 snap 0 seq_num 270) 2014-07-01 16:54:29,492.492 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:54:29,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 16:54:30,229.229 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:54:31,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:826: done (6 left) 2014-07-01 16:54:31,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:828: done (5 left) 2014-07-01 16:54:31,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:829: done (4 left) 2014-07-01 16:54:31,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:831: done (3 left) 2014-07-01 16:54:31,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:832: done (2 left) 2014-07-01 16:54:31,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:833: done (1 left) 2014-07-01 16:54:31,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:834: done (0 left) 2014-07-01 16:54:31,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:835: snap_remove snap 39 2014-07-01 16:54:31,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:835: done (0 left) 2014-07-01 16:54:31,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:836: read oid 165 snap -1 2014-07-01 16:54:31,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:837: read oid 38 snap -1 2014-07-01 16:54:31,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: write oid 295 current snap is 53 2014-07-01 16:54:31,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: seq_num 626 ranges {614779=658158,1414778=1} 2014-07-01 16:54:31,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: writing mira0437250-295 from 614779 to 1272937 tid 1 2014-07-01 16:54:31,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: writing mira0437250-295 from 1414778 to 1414779 tid 2 2014-07-01 16:54:31,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:839: delete oid 482 current snap is 53 2014-07-01 16:54:31,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:839: done (3 left) 2014-07-01 16:54:31,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: read oid 115 snap -1 2014-07-01 16:54:31,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: snap_create 2014-07-01 16:54:31,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:836: expect (ObjNum 164 snap 0 seq_num 164) 2014-07-01 16:54:32,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: done (12 left) 2014-07-01 16:54:32,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: done (11 left) 2014-07-01 16:54:32,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: done (10 left) 2014-07-01 16:54:32,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: done (9 left) 2014-07-01 16:54:32,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: done (8 left) 2014-07-01 16:54:32,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: done (7 left) 2014-07-01 16:54:32,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2582: done (6 left) 2014-07-01 16:54:32,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2584: done (5 left) 2014-07-01 16:54:32,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: done (4 left) 2014-07-01 16:54:32,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: done (3 left) 2014-07-01 16:54:32,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: done (2 left) 2014-07-01 16:54:32,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: done (1 left) 2014-07-01 16:54:32,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2590: done (0 left) 2014-07-01 16:54:32,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 440 current snap is 0 2014-07-01 16:54:32,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: seq_num 1652 ranges {3072000=782336,3854336=479232,4333568=446464,4780032=679936} 2014-07-01 16:54:32,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: writing mira0435601-440 from 3072000 to 3854336 tid 1 2014-07-01 16:54:32,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: writing mira0435601-440 from 3854336 to 4333568 tid 2 2014-07-01 16:54:32,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: writing mira0435601-440 from 4333568 to 4780032 tid 3 2014-07-01 16:54:32,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: writing mira0435601-440 from 4780032 to 5459968 tid 4 2014-07-01 16:54:32,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2592: read oid 288 snap -1 2014-07-01 16:54:32,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 1 to mira0435601-440 2014-07-01 16:54:32,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 2 to mira0435601-440 2014-07-01 16:54:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 3 to mira0435601-440 2014-07-01 16:54:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 436 current snap is 0 2014-07-01 16:54:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: seq_num 1653 ranges {4108288=651264,4759552=413696,5173248=671744,5844992=520192} 2014-07-01 16:54:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0435601-436 from 4108288 to 4759552 tid 1 2014-07-01 16:54:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0435601-436 from 4759552 to 5173248 tid 2 2014-07-01 16:54:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0435601-436 from 5173248 to 5844992 tid 3 2014-07-01 16:54:32,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0435601-436 from 5844992 to 6365184 tid 4 2014-07-01 16:54:32,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 16:54:32,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: seq_num 1654 ranges {10813440=679936,11493376=401408} 2014-07-01 16:54:32,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: writing mira0435601-337 from 10813440 to 11493376 tid 1 2014-07-01 16:54:32,964.964 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:54:32,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: writing mira0435601-337 from 11493376 to 11894784 tid 2 2014-07-01 16:54:32,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 174 current snap is 0 2014-07-01 16:54:32,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: seq_num 1655 ranges {3784704=602112,4386816=630784,5017600=573440} 2014-07-01 16:54:32,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0435601-174 from 3784704 to 4386816 tid 1 2014-07-01 16:54:32,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0435601-174 from 4386816 to 5017600 tid 2 2014-07-01 16:54:32,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0435601-174 from 5017600 to 5591040 tid 3 2014-07-01 16:54:32,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:2596: delete oid 342 current snap is 0 2014-07-01 16:54:32,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 4 to mira0435601-440 2014-07-01 16:54:32,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 6 to mira0435601-440 2014-07-01 16:54:32,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: finishing write tid 7 to mira0435601-440 2014-07-01 16:54:32,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 147 (ObjNum 1652 snap 0 seq_num 1652) dirty exists 2014-07-01 16:54:32,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 1 to mira0435601-436 2014-07-01 16:54:32,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2592: expect (ObjNum 1089 snap 0 seq_num 1089) 2014-07-01 16:54:33,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: finishing write tid 1 to mira0437250-295 2014-07-01 16:54:33,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: finishing write tid 2 to mira0437250-295 2014-07-01 16:54:33,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: finishing write tid 4 to mira0437250-295 2014-07-01 16:54:33,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: finishing write tid 5 to mira0437250-295 2014-07-01 16:54:33,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 225 (ObjNum 626 snap 53 seq_num 626) dirty exists 2014-07-01 16:54:33,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: expect (ObjNum 114 snap 0 seq_num 114) 2014-07-01 16:54:33,654.654 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 6, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.018212', u'skew': u'0.000729', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.008837', u'skew': u'0.000669', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:53:44.723074', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882126044, u'health': u'HEALTH_OK', u'kb_used': 14655008, u'store_stats': {u'bytes_total': 8793077, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4598757}}, {u'last_updated': u'2014-07-01 07:53:54.540523', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886208180, u'health': u'HEALTH_OK', u'kb_used': 10572872, u'store_stats': {u'bytes_total': 12986371, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4597747}}, {u'last_updated': u'2014-07-01 07:54:11.323581', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886198504, u'health': u'HEALTH_OK', u'kb_used': 10582548, u'store_stats': {u'bytes_total': 6539188, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4442020}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 434, u'data_bytes': 2329391461, u'bytes_used': 4915007488, u'version': 1862, u'pgs_by_state': [{u'count': 434, u'state_name': u'active+clean'}], u'bytes_avail': 5993384038400}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 731, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:54:33,655.655 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:54:33,655.655 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:33,656.656 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 3 to 0.338682723899 2014-07-01 16:54:33,656.656 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 0.338682723899' 2014-07-01 16:54:34,799.799 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 0.338683 (8221952) 2014-07-01 16:54:35,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:837: expect (ObjNum 37 snap 0 seq_num 37) 2014-07-01 16:54:35,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:836: done (4 left) 2014-07-01 16:54:35,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:837: done (3 left) 2014-07-01 16:54:35,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:838: done (2 left) 2014-07-01 16:54:35,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:840: done (1 left) 2014-07-01 16:54:35,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:841: done (0 left) 2014-07-01 16:54:35,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: snap_create 2014-07-01 16:54:35,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:842: done (0 left) 2014-07-01 16:54:35,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:843: read oid 35 snap -1 2014-07-01 16:54:35,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:844: snap_create 2014-07-01 16:54:35,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:843: expect (ObjNum 34 snap 0 seq_num 34) 2014-07-01 16:54:36,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 2 to mira0435601-436 2014-07-01 16:54:36,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 3 to mira0435601-436 2014-07-01 16:54:36,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 4 to mira0435601-436 2014-07-01 16:54:36,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 6 to mira0435601-436 2014-07-01 16:54:36,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 7 to mira0435601-436 2014-07-01 16:54:36,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 436 v 233 (ObjNum 1653 snap 0 seq_num 1653) dirty exists 2014-07-01 16:54:36,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 1 to mira0435601-337 2014-07-01 16:54:36,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: done (5 left) 2014-07-01 16:54:36,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2592: done (4 left) 2014-07-01 16:54:36,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: done (3 left) 2014-07-01 16:54:36,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2596: done (2 left) 2014-07-01 16:54:36,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 241 current snap is 0 2014-07-01 16:54:36,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: seq_num 1657 ranges {8134656=499712} 2014-07-01 16:54:36,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:843: done (1 left) 2014-07-01 16:54:36,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:844: done (0 left) 2014-07-01 16:54:36,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: read oid 264 snap -1 2014-07-01 16:54:36,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: delete oid 467 current snap is 56 2014-07-01 16:54:36,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: expect (ObjNum 263 snap 0 seq_num 263) 2014-07-01 16:54:36,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: writing mira0435601-241 from 8134656 to 8634368 tid 1 2014-07-01 16:54:36,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2598: read oid 103 snap -1 2014-07-01 16:54:36,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 2 to mira0435601-337 2014-07-01 16:54:36,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 4 to mira0435601-337 2014-07-01 16:54:36,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 5 to mira0435601-337 2014-07-01 16:54:36,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 351 (ObjNum 1654 snap 0 seq_num 1654) dirty exists 2014-07-01 16:54:36,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 1 to mira0435601-174 2014-07-01 16:54:36,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: done (3 left) 2014-07-01 16:54:36,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2599: read oid 153 snap -1 2014-07-01 16:54:36,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 2 to mira0435601-174 2014-07-01 16:54:36,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 3 to mira0435601-174 2014-07-01 16:54:36,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 5 to mira0435601-174 2014-07-01 16:54:36,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 6 to mira0435601-174 2014-07-01 16:54:36,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 262 (ObjNum 1655 snap 0 seq_num 1655) dirty exists 2014-07-01 16:54:36,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: done (3 left) 2014-07-01 16:54:36,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2600: read oid 430 snap -1 2014-07-01 16:54:36,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 16 current snap is 0 2014-07-01 16:54:36,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: seq_num 1658 ranges {1691648=372736} 2014-07-01 16:54:36,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: writing mira0435601-16 from 1691648 to 2064384 tid 1 2014-07-01 16:54:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: read oid 53 snap -1 2014-07-01 16:54:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 422 current snap is 0 2014-07-01 16:54:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: seq_num 1659 ranges {1642496=692224,2334720=401408} 2014-07-01 16:54:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: writing mira0435601-422 from 1642496 to 2334720 tid 1 2014-07-01 16:54:36,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: writing mira0435601-422 from 2334720 to 2736128 tid 2 2014-07-01 16:54:36,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2604: read oid 268 snap -1 2014-07-01 16:54:36,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2605: read oid 83 snap -1 2014-07-01 16:54:36,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2606: read oid 443 snap -1 2014-07-01 16:54:36,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2607: read oid 107 snap -1 2014-07-01 16:54:36,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 321 current snap is 0 2014-07-01 16:54:36,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: seq_num 1660 ranges {3268608=720896,3989504=684032,4673536=741376,5414912=180224} 2014-07-01 16:54:36,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: writing mira0435601-321 from 3268608 to 3989504 tid 1 2014-07-01 16:54:36,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: writing mira0435601-321 from 3989504 to 4673536 tid 2 2014-07-01 16:54:36,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: writing mira0435601-321 from 4673536 to 5414912 tid 3 2014-07-01 16:54:36,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: writing mira0435601-321 from 5414912 to 5595136 tid 4 2014-07-01 16:54:36,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 82 current snap is 0 2014-07-01 16:54:36,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: seq_num 1661 ranges {4501504=589824,5091328=622592,5713920=409600,6123520=503808,6627328=69632} 2014-07-01 16:54:37,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: writing mira0435601-82 from 4501504 to 5091328 tid 1 2014-07-01 16:54:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: writing mira0435601-82 from 5091328 to 5713920 tid 2 2014-07-01 16:54:37,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: writing mira0435601-82 from 5713920 to 6123520 tid 3 2014-07-01 16:54:37,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: writing mira0435601-82 from 6123520 to 6627328 tid 4 2014-07-01 16:54:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: writing mira0435601-82 from 6627328 to 6696960 tid 5 2014-07-01 16:54:37,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: delete oid 281 current snap is 0 2014-07-01 16:54:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2599: expect (ObjNum 1599 snap 0 seq_num 1599) 2014-07-01 16:54:37,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:845: done (1 left) 2014-07-01 16:54:37,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:846: done (0 left) 2014-07-01 16:54:37,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:847: snap_remove snap 48 2014-07-01 16:54:37,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:847: done (0 left) 2014-07-01 16:54:37,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: snap_remove snap 37 2014-07-01 16:54:38,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:848: done (0 left) 2014-07-01 16:54:38,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: write oid 389 current snap is 56 2014-07-01 16:54:38,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: seq_num 629 ranges {716020=609667,1903160=690802,3090327=25693} 2014-07-01 16:54:38,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: writing mira0437250-389 from 716020 to 1325687 tid 1 2014-07-01 16:54:38,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: writing mira0437250-389 from 1903160 to 2593962 tid 2 2014-07-01 16:54:38,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: writing mira0437250-389 from 3090327 to 3116020 tid 3 2014-07-01 16:54:38,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:850: snap_create 2014-07-01 16:54:39,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: finishing write tid 1 to mira0437250-389 2014-07-01 16:54:39,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: finishing write tid 2 to mira0437250-389 2014-07-01 16:54:39,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: finishing write tid 3 to mira0437250-389 2014-07-01 16:54:39,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: finishing write tid 5 to mira0437250-389 2014-07-01 16:54:39,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: finishing write tid 6 to mira0437250-389 2014-07-01 16:54:39,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 197 (ObjNum 629 snap 56 seq_num 629) dirty exists 2014-07-01 16:54:39,811.811 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:39,811.811 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:39,812.812 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:54:39,812.812 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:54:39,812.812 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:54:39,813.813 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:54:39,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:849: done (1 left) 2014-07-01 16:54:39,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:850: done (0 left) 2014-07-01 16:54:39,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: write oid 321 current snap is 57 2014-07-01 16:54:39,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: seq_num 630 ranges {659826=732895,2015244=609573,3386656=473170} 2014-07-01 16:54:39,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: writing mira0437250-321 from 659826 to 1392721 tid 1 2014-07-01 16:54:39,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: writing mira0437250-321 from 2015244 to 2624817 tid 2 2014-07-01 16:54:39,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: writing mira0437250-321 from 3386656 to 3859826 tid 3 2014-07-01 16:54:39,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:852: rollback oid 357 current snap is 57 2014-07-01 16:54:39,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 357 to 49 2014-07-01 16:54:39,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: read oid 282 snap -1 2014-07-01 16:54:39,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:854: snap_create 2014-07-01 16:54:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 298 (ObjNum 356 snap 0 seq_num 356) dirty exists 2014-07-01 16:54:40,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: finishing write tid 1 to mira0437250-321 2014-07-01 16:54:40,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: finishing write tid 2 to mira0437250-321 2014-07-01 16:54:40,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: finishing write tid 3 to mira0437250-321 2014-07-01 16:54:40,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: finishing write tid 5 to mira0437250-321 2014-07-01 16:54:40,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: finishing write tid 6 to mira0437250-321 2014-07-01 16:54:40,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 304 (ObjNum 630 snap 57 seq_num 630) dirty exists 2014-07-01 16:54:40,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:2605: expect (ObjNum 1509 snap 0 seq_num 1509) 2014-07-01 16:54:40,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: expect (ObjNum 281 snap 0 seq_num 281) 2014-07-01 16:54:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:851: done (3 left) 2014-07-01 16:54:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:852: done (2 left) 2014-07-01 16:54:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:853: done (1 left) 2014-07-01 16:54:40,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:854: done (0 left) 2014-07-01 16:54:40,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: snap_remove snap 44 2014-07-01 16:54:41,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: expect (ObjNum 1549 snap 0 seq_num 1549) 2014-07-01 16:54:41,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:855: done (0 left) 2014-07-01 16:54:41,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: rollback oid 304 current snap is 58 2014-07-01 16:54:41,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 304 to 46 2014-07-01 16:54:41,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: write oid 104 current snap is 58 2014-07-01 16:54:41,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: seq_num 631 ranges {603631=1} 2014-07-01 16:54:41,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: writing mira0437250-104 from 603631 to 603632 tid 1 2014-07-01 16:54:41,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:858: snap_create 2014-07-01 16:54:41,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 304 v 162 (ObjNum 303 snap 0 seq_num 303) dirty exists 2014-07-01 16:54:41,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:2604: expect (ObjNum 566 snap 0 seq_num 566) 2014-07-01 16:54:42,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: finishing write tid 1 to mira0437250-104 2014-07-01 16:54:42,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: finishing write tid 3 to mira0437250-104 2014-07-01 16:54:42,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: finishing write tid 4 to mira0437250-104 2014-07-01 16:54:42,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 104 v 299 (ObjNum 631 snap 58 seq_num 631) dirty exists 2014-07-01 16:54:42,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:856: done (2 left) 2014-07-01 16:54:42,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:857: done (1 left) 2014-07-01 16:54:42,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:858: done (0 left) 2014-07-01 16:54:42,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: write oid 40 current snap is 59 2014-07-01 16:54:42,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: seq_num 632 ranges {416238=400000} 2014-07-01 16:54:42,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: writing mira0437250-40 from 416238 to 816238 tid 1 2014-07-01 16:54:42,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: snap_create 2014-07-01 16:54:43,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 1 to mira0437250-40 2014-07-01 16:54:43,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 3 to mira0437250-40 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: finishing write tid 1 to mira0435601-16 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: finishing write tid 3 to mira0435601-16 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 1 to mira0435601-321 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 2 to mira0435601-321 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 3 to mira0435601-321 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 4 to mira0435601-321 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 6 to mira0435601-321 2014-07-01 16:54:43,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2598: expect (ObjNum 1643 snap 0 seq_num 1643) 2014-07-01 16:54:43,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:860: done (1 left) 2014-07-01 16:54:43,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: write oid 287 current snap is 60 2014-07-01 16:54:43,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: seq_num 633 ranges {608619=532980,1867272=454206,2608618=1} 2014-07-01 16:54:43,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: writing mira0437250-287 from 608619 to 1141599 tid 1 2014-07-01 16:54:43,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: writing mira0437250-287 from 1867272 to 2321478 tid 2 2014-07-01 16:54:43,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: writing mira0437250-287 from 2608618 to 2608619 tid 3 2014-07-01 16:54:43,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: read oid 196 snap 46 2014-07-01 16:54:43,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: finishing write tid 4 to mira0437250-40 2014-07-01 16:54:43,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 272 (ObjNum 632 snap 59 seq_num 632) dirty exists 2014-07-01 16:54:43,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:859: done (2 left) 2014-07-01 16:54:43,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: write oid 451 current snap is 60 2014-07-01 16:54:43,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: seq_num 634 ranges {704873=656208,1849180=455693} 2014-07-01 16:54:43,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: writing mira0437250-451 from 704873 to 1361081 tid 1 2014-07-01 16:54:43,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: writing mira0437250-451 from 1849180 to 2304873 tid 2 2014-07-01 16:54:44,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: write oid 274 current snap is 60 2014-07-01 16:54:44,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: seq_num 635 ranges {401127=628013,1811890=189237} 2014-07-01 16:54:44,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0437250-274 from 401127 to 1029140 tid 1 2014-07-01 16:54:44,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: writing mira0437250-274 from 1811890 to 2001127 tid 2 2014-07-01 16:54:44,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:865: delete oid 157 current snap is 60 2014-07-01 16:54:44,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 1 to mira0437250-287 2014-07-01 16:54:44,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 2 to mira0437250-287 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 3 to mira0437250-287 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 5 to mira0437250-287 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: finishing write tid 6 to mira0437250-287 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 211 (ObjNum 633 snap 60 seq_num 633) dirty exists 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 1 to mira0437250-451 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 2 to mira0437250-451 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 4 to mira0437250-451 2014-07-01 16:54:44,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: finishing write tid 5 to mira0437250-451 2014-07-01 16:54:44,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 218 (ObjNum 634 snap 60 seq_num 634) dirty exists 2014-07-01 16:54:44,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: expect (ObjNum 195 snap 0 seq_num 195) 2014-07-01 16:54:44,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:2600: expect (ObjNum 1587 snap 0 seq_num 1587) 2014-07-01 16:54:45,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 1 to mira0437250-274 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 2 to mira0437250-274 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 4 to mira0437250-274 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: finishing write tid 5 to mira0437250-274 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 200 (ObjNum 635 snap 60 seq_num 635) dirty exists 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:861: done (4 left) 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:862: done (3 left) 2014-07-01 16:54:45,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:863: done (2 left) 2014-07-01 16:54:45,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:864: done (1 left) 2014-07-01 16:54:45,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:865: done (0 left) 2014-07-01 16:54:45,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:866: snap_remove snap 47 2014-07-01 16:54:45,281.281 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:45,282.282 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:45,282.282 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:54:45,282.282 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:54:45,283.283 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:54:45,283.283 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:54:45,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:866: done (0 left) 2014-07-01 16:54:45,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: read oid 44 snap -1 2014-07-01 16:54:45,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: snap_create 2014-07-01 16:54:45,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: expect (ObjNum 43 snap 0 seq_num 43) 2014-07-01 16:54:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:867: done (1 left) 2014-07-01 16:54:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:868: done (0 left) 2014-07-01 16:54:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: snap_create 2014-07-01 16:54:47,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:869: done (0 left) 2014-07-01 16:54:47,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:870: snap_remove snap 46 2014-07-01 16:54:47,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2606: expect (ObjNum 1521 snap 0 seq_num 1521) 2014-07-01 16:54:48,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:870: done (0 left) 2014-07-01 16:54:48,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: rollback oid 483 current snap is 62 2014-07-01 16:54:48,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 483 to 57 2014-07-01 16:54:48,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:872: snap_remove snap 45 2014-07-01 16:54:48,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 291 (ObjNum 482 snap 0 seq_num 482) dirty exists 2014-07-01 16:54:49,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2607: expect (ObjNum 1632 snap 0 seq_num 1632) 2014-07-01 16:54:49,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:871: done (1 left) 2014-07-01 16:54:49,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:872: done (0 left) 2014-07-01 16:54:49,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:873: rollback oid 307 current snap is 62 2014-07-01 16:54:49,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 307 to 54 2014-07-01 16:54:49,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: write oid 101 current snap is 62 2014-07-01 16:54:49,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: seq_num 637 ranges {6113=1} 2014-07-01 16:54:49,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: writing mira0437250-101 from 6113 to 6114 tid 1 2014-07-01 16:54:49,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:875: snap_create 2014-07-01 16:54:49,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: finishing write tid 1 to mira0437250-101 2014-07-01 16:54:49,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: finishing write tid 3 to mira0437250-101 2014-07-01 16:54:49,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: finishing write tid 4 to mira0437250-101 2014-07-01 16:54:49,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 220 (ObjNum 637 snap 62 seq_num 637) dirty exists 2014-07-01 16:54:49,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 0 (ObjNum 39410720 snap 0 seq_num 0) dirty dne 2014-07-01 16:54:50,757.757 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 4, 2, 0, 5, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:50,757.757 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:50,758.758 INFO:teuthology.task.thrashosds.thrasher:Removing osd 4, in_osds are: [3, 4, 2, 0, 5, 1] 2014-07-01 16:54:50,758.758 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 4' 2014-07-01 16:54:50,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: finishing write tid 4 to mira0435601-16 2014-07-01 16:54:50,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 207 (ObjNum 1658 snap 0 seq_num 1658) dirty exists 2014-07-01 16:54:50,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: finishing write tid 7 to mira0435601-321 2014-07-01 16:54:50,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 212 (ObjNum 1660 snap 0 seq_num 1660) dirty exists 2014-07-01 16:54:50,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: finishing write tid 1 to mira0435601-422 2014-07-01 16:54:50,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: finishing write tid 2 to mira0435601-422 2014-07-01 16:54:50,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: finishing write tid 4 to mira0435601-422 2014-07-01 16:54:50,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 1 to mira0435601-82 2014-07-01 16:54:50,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 2 to mira0435601-82 2014-07-01 16:54:50,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 3 to mira0435601-82 2014-07-01 16:54:50,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2598: done (13 left) 2014-07-01 16:54:50,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2599: done (12 left) 2014-07-01 16:54:50,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2600: done (11 left) 2014-07-01 16:54:50,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: done (10 left) 2014-07-01 16:54:50,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: done (9 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2604: done (8 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2605: done (7 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2606: done (6 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2607: done (5 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: done (4 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: done (3 left) 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 164 current snap is 0 2014-07-01 16:54:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: seq_num 1663 ranges {1667072=491520,2158592=401408} 2014-07-01 16:54:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: writing mira0435601-164 from 1667072 to 2158592 tid 1 2014-07-01 16:54:50,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: writing mira0435601-164 from 2158592 to 2560000 tid 2 2014-07-01 16:54:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: read oid 460 snap -1 2014-07-01 16:54:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: finishing write tid 5 to mira0435601-422 2014-07-01 16:54:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 236 (ObjNum 1659 snap 0 seq_num 1659) dirty exists 2014-07-01 16:54:51,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 4 to mira0435601-82 2014-07-01 16:54:51,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 5 to mira0435601-82 2014-07-01 16:54:51,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: done (4 left) 2014-07-01 16:54:51,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: read oid 364 snap -1 2014-07-01 16:54:51,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 7 to mira0435601-82 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: finishing write tid 8 to mira0435601-82 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 342 (ObjNum 1661 snap 0 seq_num 1661) dirty exists 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: finishing write tid 1 to mira0435601-241 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: finishing write tid 3 to mira0435601-241 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: finishing write tid 4 to mira0435601-241 2014-07-01 16:54:51,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 225 (ObjNum 1657 snap 0 seq_num 1657) dirty exists 2014-07-01 16:54:51,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: done (4 left) 2014-07-01 16:54:51,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: done (3 left) 2014-07-01 16:54:51,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2614: read oid 343 snap -1 2014-07-01 16:54:51,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2615: delete oid 245 current snap is 0 2014-07-01 16:54:51,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: finishing write tid 1 to mira0435601-164 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2615: done (4 left) 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2616: read oid 331 snap -1 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2617: delete oid 498 current snap is 0 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2616: done (5 left) 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2617: done (4 left) 2014-07-01 16:54:51,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 179 current snap is 0 2014-07-01 16:54:51,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: seq_num 1666 ranges {4317184=311296} 2014-07-01 16:54:51,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: writing mira0435601-179 from 4317184 to 4628480 tid 1 2014-07-01 16:54:51,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:54:51,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: seq_num 1667 ranges {5783552=737280,6520832=413696,6934528=389120} 2014-07-01 16:54:51,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: writing mira0435601-27 from 5783552 to 6520832 tid 1 2014-07-01 16:54:51,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: writing mira0435601-27 from 6520832 to 6934528 tid 2 2014-07-01 16:54:51,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: writing mira0435601-27 from 6934528 to 7323648 tid 3 2014-07-01 16:54:51,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2620: read oid 283 snap -1 2014-07-01 16:54:51,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: finishing write tid 2 to mira0435601-164 2014-07-01 16:54:51,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: finishing write tid 4 to mira0435601-164 2014-07-01 16:54:51,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: expect (ObjNum 1489 snap 0 seq_num 1489) 2014-07-01 16:54:51,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:873: done (2 left) 2014-07-01 16:54:51,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:874: done (1 left) 2014-07-01 16:54:51,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:875: done (0 left) 2014-07-01 16:54:51,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: read oid 260 snap -1 2014-07-01 16:54:51,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:877: snap_create 2014-07-01 16:54:51,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: expect (ObjNum 259 snap 0 seq_num 259) 2014-07-01 16:54:51,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:876: done (1 left) 2014-07-01 16:54:51,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:877: done (0 left) 2014-07-01 16:54:51,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: write oid 217 current snap is 64 2014-07-01 16:54:51,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: seq_num 638 ranges {500320=730946,1716142=708030,3223787=476533} 2014-07-01 16:54:51,994.994 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.4. 2014-07-01 16:54:52,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0437250-217 from 500320 to 1231266 tid 1 2014-07-01 16:54:52,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0437250-217 from 1716142 to 2424172 tid 2 2014-07-01 16:54:52,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: writing mira0437250-217 from 3223787 to 3700320 tid 3 2014-07-01 16:54:52,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:879: delete oid 389 current snap is 64 2014-07-01 16:54:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2614: expect (ObjNum 1495 snap 0 seq_num 1495) 2014-07-01 16:54:53,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 1 to mira0437250-217 2014-07-01 16:54:53,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 2 to mira0437250-217 2014-07-01 16:54:53,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 3 to mira0437250-217 2014-07-01 16:54:53,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 5 to mira0437250-217 2014-07-01 16:54:53,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: finishing write tid 5 to mira0435601-164 2014-07-01 16:54:53,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 267 (ObjNum 1663 snap 0 seq_num 1663) dirty exists 2014-07-01 16:54:53,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: expect (ObjNum 1635 snap 0 seq_num 1635) 2014-07-01 16:54:53,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: finishing write tid 6 to mira0437250-217 2014-07-01 16:54:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 218 (ObjNum 638 snap 64 seq_num 638) dirty exists 2014-07-01 16:54:53,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:878: done (1 left) 2014-07-01 16:54:53,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:879: done (0 left) 2014-07-01 16:54:53,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:880: snap_create 2014-07-01 16:54:54,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:880: done (0 left) 2014-07-01 16:54:54,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: rollback oid 172 current snap is 65 2014-07-01 16:54:54,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 172 to 57 2014-07-01 16:54:54,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: write oid 231 current snap is 65 2014-07-01 16:54:54,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: seq_num 640 ranges {789979=400000} 2014-07-01 16:54:54,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: writing mira0437250-231 from 789979 to 1189979 tid 1 2014-07-01 16:54:54,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: rollback oid 55 current snap is 65 2014-07-01 16:54:54,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 55 to 53 2014-07-01 16:54:54,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: delete oid 20 current snap is 65 2014-07-01 16:54:54,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:884: done (3 left) 2014-07-01 16:54:54,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: read oid 204 snap -1 2014-07-01 16:54:54,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: read oid 238 snap -1 2014-07-01 16:54:54,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: read oid 176 snap -1 2014-07-01 16:54:54,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: write oid 68 current snap is 65 2014-07-01 16:54:54,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: seq_num 642 ranges {546391=405683,1551102=595289} 2014-07-01 16:54:54,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: writing mira0437250-68 from 546391 to 952074 tid 1 2014-07-01 16:54:54,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: writing mira0437250-68 from 1551102 to 2146391 tid 2 2014-07-01 16:54:54,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: snap_create 2014-07-01 16:54:54,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: expect (ObjNum 237 snap 0 seq_num 237) 2014-07-01 16:54:55,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: expect (ObjNum 203 snap 0 seq_num 203) 2014-07-01 16:54:55,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: finishing write tid 1 to mira0435601-179 2014-07-01 16:54:55,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: finishing write tid 3 to mira0435601-179 2014-07-01 16:54:55,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: finishing write tid 4 to mira0435601-179 2014-07-01 16:54:55,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 179 v 221 (ObjNum 1666 snap 0 seq_num 1666) dirty exists 2014-07-01 16:54:55,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: finishing write tid 1 to mira0435601-27 2014-07-01 16:54:55,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: finishing write tid 2 to mira0435601-27 2014-07-01 16:54:55,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: done (6 left) 2014-07-01 16:54:55,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: done (5 left) 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: done (4 left) 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2614: done (3 left) 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: done (2 left) 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2621: delete oid 28 current snap is 0 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: finishing write tid 3 to mira0435601-27 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: finishing write tid 5 to mira0435601-27 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: finishing write tid 6 to mira0435601-27 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 225 (ObjNum 1667 snap 0 seq_num 1667) dirty exists 2014-07-01 16:54:55,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2620: expect (ObjNum 1461 snap 0 seq_num 1461) 2014-07-01 16:54:55,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: done (2 left) 2014-07-01 16:54:55,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2620: done (1 left) 2014-07-01 16:54:55,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2621: done (0 left) 2014-07-01 16:54:55,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:54:55,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: seq_num 1669 ranges {4694016=532480,5226496=581632,5808128=507904,6316032=516096} 2014-07-01 16:54:55,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: writing mira0435601-200 from 4694016 to 5226496 tid 1 2014-07-01 16:54:55,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: writing mira0435601-200 from 5226496 to 5808128 tid 2 2014-07-01 16:54:55,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: writing mira0435601-200 from 5808128 to 6316032 tid 3 2014-07-01 16:54:55,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: writing mira0435601-200 from 6316032 to 6832128 tid 4 2014-07-01 16:54:55,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: read oid 451 snap -1 2014-07-01 16:54:55,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2624: read oid 112 snap -1 2014-07-01 16:54:55,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 22 current snap is 0 2014-07-01 16:54:55,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: seq_num 1670 ranges {2703360=557056,3260416=401408} 2014-07-01 16:54:56,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: writing mira0435601-22 from 2703360 to 3260416 tid 1 2014-07-01 16:54:56,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: writing mira0435601-22 from 3260416 to 3661824 tid 2 2014-07-01 16:54:56,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 177 current snap is 0 2014-07-01 16:54:56,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: seq_num 1671 ranges {4313088=675840,4988928=643072,5632000=405504,6037504=155648} 2014-07-01 16:54:56,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: writing mira0435601-177 from 4313088 to 4988928 tid 1 2014-07-01 16:54:56,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: writing mira0435601-177 from 4988928 to 5632000 tid 2 2014-07-01 16:54:56,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: writing mira0435601-177 from 5632000 to 6037504 tid 3 2014-07-01 16:54:56,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: writing mira0435601-177 from 6037504 to 6193152 tid 4 2014-07-01 16:54:56,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 284 current snap is 0 2014-07-01 16:54:56,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: seq_num 1672 ranges {3248128=704512} 2014-07-01 16:54:56,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: writing mira0435601-284 from 3248128 to 3952640 tid 1 2014-07-01 16:54:56,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:2628: read oid 390 snap -1 2014-07-01 16:54:56,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 1 to mira0435601-200 2014-07-01 16:54:56,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 2 to mira0435601-200 2014-07-01 16:54:56,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 3 to mira0435601-200 2014-07-01 16:54:56,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: expect (ObjNum 1488 snap 0 seq_num 1488) 2014-07-01 16:54:56,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:2628: expect (ObjNum 389 snap 0 seq_num 389) 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 4 to mira0435601-200 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 6 to mira0435601-200 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 1 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 2 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 3 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 4 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 6 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: finishing write tid 7 to mira0435601-200 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 127 (ObjNum 1669 snap 0 seq_num 1669) dirty exists 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: finishing write tid 7 to mira0435601-177 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 144 (ObjNum 1671 snap 0 seq_num 1671) dirty exists 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: done (6 left) 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: done (5 left) 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2624: done (4 left) 2014-07-01 16:54:56,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: done (3 left) 2014-07-01 16:54:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2628: done (2 left) 2014-07-01 16:54:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: read oid 329 snap -1 2014-07-01 16:54:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2630: read oid 308 snap -1 2014-07-01 16:54:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2631: read oid 59 snap -1 2014-07-01 16:54:56,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2632: read oid 205 snap -1 2014-07-01 16:54:56,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:2633: delete oid 82 current snap is 0 2014-07-01 16:54:56,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: expect (ObjNum 175 snap 0 seq_num 175) 2014-07-01 16:54:57,020.020 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:54:57,020.020 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:54:57,020.020 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2014-07-01 16:54:57,020.020 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 1' 2014-07-01 16:54:57,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: finishing write tid 1 to mira0437250-68 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: finishing write tid 2 to mira0437250-68 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: finishing write tid 4 to mira0437250-68 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: finishing write tid 5 to mira0437250-68 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 241 (ObjNum 642 snap 65 seq_num 642) dirty exists 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 1 to mira0437250-231 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 3 to mira0437250-231 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: finishing write tid 4 to mira0437250-231 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 171 (ObjNum 640 snap 65 seq_num 640) dirty exists 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 297 (ObjNum 54 snap 0 seq_num 54) dirty exists 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 226 (ObjNum 171 snap 0 seq_num 171) dirty exists 2014-07-01 16:54:57,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:881: done (7 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:882: done (6 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:883: done (5 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:885: done (4 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:886: done (3 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:887: done (2 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:888: done (1 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:889: done (0 left) 2014-07-01 16:54:57,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: write oid 185 current snap is 66 2014-07-01 16:54:57,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: seq_num 643 ranges {490197=627088,1762387=596926,2861338=28859} 2014-07-01 16:54:57,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: writing mira0437250-185 from 490197 to 1117285 tid 1 2014-07-01 16:54:57,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: writing mira0437250-185 from 1762387 to 2359313 tid 2 2014-07-01 16:54:57,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: writing mira0437250-185 from 2861338 to 2890197 tid 3 2014-07-01 16:54:57,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: rollback oid 246 current snap is 66 2014-07-01 16:54:57,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 246 to 31 2014-07-01 16:54:57,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: snap_create 2014-07-01 16:54:58,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:892: done (2 left) 2014-07-01 16:54:58,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: write oid 190 current snap is 67 2014-07-01 16:54:58,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: seq_num 644 ranges {586450=633964,1876518=515698,2586449=1} 2014-07-01 16:54:58,079.079 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 1 (8655362) 2014-07-01 16:54:58,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: writing mira0437250-190 from 586450 to 1220414 tid 1 2014-07-01 16:54:58,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: writing mira0437250-190 from 1876518 to 2392216 tid 2 2014-07-01 16:54:58,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: writing mira0437250-190 from 2586449 to 2586450 tid 3 2014-07-01 16:54:58,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: write oid 343 current snap is 67 2014-07-01 16:54:58,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: seq_num 645 ranges {647632=605768,1954554=737317,3477293=370339} 2014-07-01 16:54:58,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: writing mira0437250-343 from 647632 to 1253400 tid 1 2014-07-01 16:54:58,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: writing mira0437250-343 from 1954554 to 2691871 tid 2 2014-07-01 16:54:58,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: writing mira0437250-343 from 3477293 to 3847632 tid 3 2014-07-01 16:54:58,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: write oid 474 current snap is 67 2014-07-01 16:54:58,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: seq_num 646 ranges {590413=1} 2014-07-01 16:54:58,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: writing mira0437250-474 from 590413 to 590414 tid 1 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: write oid 27 current snap is 67 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: seq_num 647 ranges {286667=1} 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: writing mira0437250-27 from 286667 to 286668 tid 1 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:897: snap_create 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: finishing write tid 1 to mira0437250-190 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: finishing write tid 2 to mira0437250-190 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: finishing write tid 3 to mira0437250-190 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: finishing write tid 5 to mira0437250-190 2014-07-01 16:54:58,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: finishing write tid 6 to mira0437250-190 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 198 (ObjNum 644 snap 67 seq_num 644) dirty exists 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 1 to mira0437250-27 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 3 to mira0437250-27 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: finishing write tid 1 to mira0437250-343 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: finishing write tid 2 to mira0437250-343 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: finishing write tid 3 to mira0437250-343 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: finishing write tid 5 to mira0437250-343 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: finishing write tid 4 to mira0437250-27 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 244 (ObjNum 647 snap 67 seq_num 647) dirty exists 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: finishing write tid 6 to mira0437250-343 2014-07-01 16:54:58,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 335 (ObjNum 645 snap 67 seq_num 645) dirty exists 2014-07-01 16:54:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: finishing write tid 1 to mira0435601-22 2014-07-01 16:54:58,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: finishing write tid 2 to mira0435601-22 2014-07-01 16:54:58,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: finishing write tid 4 to mira0435601-22 2014-07-01 16:54:58,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 1 to mira0437250-185 2014-07-01 16:54:58,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 2 to mira0437250-185 2014-07-01 16:54:58,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 3 to mira0437250-185 2014-07-01 16:54:58,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 5 to mira0437250-185 2014-07-01 16:54:58,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: finishing write tid 6 to mira0437250-185 2014-07-01 16:54:58,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 231 (ObjNum 643 snap 66 seq_num 643) dirty exists 2014-07-01 16:54:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 159 (ObjNum 245 snap 0 seq_num 245) dirty exists 2014-07-01 16:54:58,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: finishing write tid 5 to mira0435601-22 2014-07-01 16:54:58,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 22 v 240 (ObjNum 1670 snap 0 seq_num 1670) dirty exists 2014-07-01 16:54:58,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2632: expect (ObjNum 204 snap 0 seq_num 204) 2014-07-01 16:54:58,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 1 to mira0437250-474 2014-07-01 16:54:58,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 3 to mira0437250-474 2014-07-01 16:54:58,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: finishing write tid 4 to mira0437250-474 2014-07-01 16:54:58,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 183 (ObjNum 646 snap 67 seq_num 646) dirty exists 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:890: done (6 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:891: done (5 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:893: done (4 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:894: done (3 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:895: done (2 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:896: done (1 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:897: done (0 left) 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:898: rollback oid 423 current snap is 68 2014-07-01 16:54:59,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 423 to 54 2014-07-01 16:54:59,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: read oid 460 snap -1 2014-07-01 16:54:59,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:900: snap_create 2014-07-01 16:54:59,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 186 (ObjNum 422 snap 0 seq_num 422) dirty exists 2014-07-01 16:54:59,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: expect (ObjNum 459 snap 0 seq_num 459) 2014-07-01 16:55:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: finishing write tid 1 to mira0435601-284 2014-07-01 16:55:00,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: finishing write tid 3 to mira0435601-284 2014-07-01 16:55:00,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: finishing write tid 4 to mira0435601-284 2014-07-01 16:55:00,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 279 (ObjNum 1672 snap 0 seq_num 1672) dirty exists 2014-07-01 16:55:00,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2630: expect (ObjNum 1314 snap 0 seq_num 1314) 2014-07-01 16:55:01,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:898: done (2 left) 2014-07-01 16:55:01,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:899: done (1 left) 2014-07-01 16:55:01,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:900: done (0 left) 2014-07-01 16:55:01,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: write oid 90 current snap is 69 2014-07-01 16:55:01,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: seq_num 648 ranges {630473=400000} 2014-07-01 16:55:01,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: writing mira0437250-90 from 630473 to 1030473 tid 1 2014-07-01 16:55:01,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: write oid 488 current snap is 69 2014-07-01 16:55:01,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: seq_num 649 ranges {575303=681530,1775302=1} 2014-07-01 16:55:01,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: expect (ObjNum 1432 snap 0 seq_num 1432) 2014-07-01 16:55:01,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: writing mira0437250-488 from 575303 to 1256833 tid 1 2014-07-01 16:55:01,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: writing mira0437250-488 from 1775302 to 1775303 tid 2 2014-07-01 16:55:01,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: snap_create 2014-07-01 16:55:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 1 to mira0437250-488 2014-07-01 16:55:01,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 2 to mira0437250-488 2014-07-01 16:55:01,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 4 to mira0437250-488 2014-07-01 16:55:01,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: finishing write tid 5 to mira0437250-488 2014-07-01 16:55:01,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 170 (ObjNum 649 snap 69 seq_num 649) dirty exists 2014-07-01 16:55:01,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: finishing write tid 1 to mira0437250-90 2014-07-01 16:55:01,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: finishing write tid 3 to mira0437250-90 2014-07-01 16:55:01,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: finishing write tid 4 to mira0437250-90 2014-07-01 16:55:01,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 90 v 338 (ObjNum 648 snap 69 seq_num 648) dirty exists 2014-07-01 16:55:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:901: done (2 left) 2014-07-01 16:55:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:902: done (1 left) 2014-07-01 16:55:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:903: done (0 left) 2014-07-01 16:55:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: write oid 388 current snap is 70 2014-07-01 16:55:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: seq_num 650 ranges {787909=405782,1655071=332838} 2014-07-01 16:55:02,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0437250-388 from 787909 to 1193691 tid 1 2014-07-01 16:55:02,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: writing mira0437250-388 from 1655071 to 1987909 tid 2 2014-07-01 16:55:02,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: snap_create 2014-07-01 16:55:02,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 1 to mira0437250-388 2014-07-01 16:55:02,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 2 to mira0437250-388 2014-07-01 16:55:02,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 4 to mira0437250-388 2014-07-01 16:55:02,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: finishing write tid 5 to mira0437250-388 2014-07-01 16:55:02,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 233 (ObjNum 650 snap 70 seq_num 650) dirty exists 2014-07-01 16:55:03,086.086 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:55:03,086.086 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:55:03,086.086 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:55:03,086.086 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:55:03,086.086 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:55:03,086.086 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:55:03,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:904: done (1 left) 2014-07-01 16:55:03,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:905: done (0 left) 2014-07-01 16:55:03,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:906: snap_remove snap 60 2014-07-01 16:55:03,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: done (6 left) 2014-07-01 16:55:03,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: done (5 left) 2014-07-01 16:55:03,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: done (4 left) 2014-07-01 16:55:03,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:2630: done (3 left) 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2631: done (2 left) 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2632: done (1 left) 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2633: done (0 left) 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2634: read oid 383 snap -1 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 431 current snap is 0 2014-07-01 16:55:03,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: seq_num 1674 ranges {2158592=495616,2654208=638976,3293184=163840} 2014-07-01 16:55:03,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: writing mira0435601-431 from 2158592 to 2654208 tid 1 2014-07-01 16:55:03,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: writing mira0435601-431 from 2654208 to 3293184 tid 2 2014-07-01 16:55:03,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: writing mira0435601-431 from 3293184 to 3457024 tid 3 2014-07-01 16:55:03,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 367 current snap is 0 2014-07-01 16:55:03,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: seq_num 1675 ranges {3158016=122880} 2014-07-01 16:55:03,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: writing mira0435601-367 from 3158016 to 3280896 tid 1 2014-07-01 16:55:04,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2637: delete oid 182 current snap is 0 2014-07-01 16:55:04,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:906: done (0 left) 2014-07-01 16:55:04,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: read oid 121 snap -1 2014-07-01 16:55:04,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:908: snap_remove snap 54 2014-07-01 16:55:04,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: expect (ObjNum 120 snap 0 seq_num 120) 2014-07-01 16:55:04,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: finishing write tid 1 to mira0435601-367 2014-07-01 16:55:04,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: finishing write tid 3 to mira0435601-367 2014-07-01 16:55:04,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2637: done (3 left) 2014-07-01 16:55:04,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 267 current snap is 0 2014-07-01 16:55:04,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: seq_num 1677 ranges {3215360=471040,3686400=790528,4476928=802816,5279744=12288} 2014-07-01 16:55:04,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: writing mira0435601-267 from 3215360 to 3686400 tid 1 2014-07-01 16:55:04,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: writing mira0435601-267 from 3686400 to 4476928 tid 2 2014-07-01 16:55:04,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: writing mira0435601-267 from 4476928 to 5279744 tid 3 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: writing mira0435601-267 from 5279744 to 5292032 tid 4 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2639: delete oid 402 current snap is 0 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2634: expect (ObjNum 957 snap 0 seq_num 957) 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:907: done (1 left) 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:908: done (0 left) 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: read oid 228 snap -1 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: write oid 137 current snap is 71 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: seq_num 651 ranges {731715=777586,2016755=543960,2731714=1} 2014-07-01 16:55:05,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: writing mira0437250-137 from 731715 to 1509301 tid 1 2014-07-01 16:55:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: writing mira0437250-137 from 2016755 to 2560715 tid 2 2014-07-01 16:55:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: writing mira0437250-137 from 2731714 to 2731715 tid 3 2014-07-01 16:55:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: delete oid 447 current snap is 71 2014-07-01 16:55:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: expect (ObjNum 227 snap 0 seq_num 227) 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 1 to mira0437250-137 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 2 to mira0437250-137 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 3 to mira0437250-137 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 5 to mira0437250-137 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: finishing write tid 6 to mira0437250-137 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 231 (ObjNum 651 snap 71 seq_num 651) dirty exists 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:909: done (2 left) 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:910: done (1 left) 2014-07-01 16:55:06,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:911: done (0 left) 2014-07-01 16:55:06,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: rollback oid 449 current snap is 71 2014-07-01 16:55:06,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 449 to 53 2014-07-01 16:55:06,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:913: read oid 44 snap -1 2014-07-01 16:55:06,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: read oid 175 snap -1 2014-07-01 16:55:06,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: write oid 302 current snap is 71 2014-07-01 16:55:06,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: seq_num 653 ranges {771774=756267,2091548=434750,2949496=222278} 2014-07-01 16:55:06,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0437250-302 from 771774 to 1528041 tid 1 2014-07-01 16:55:06,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0437250-302 from 2091548 to 2526298 tid 2 2014-07-01 16:55:06,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: writing mira0437250-302 from 2949496 to 3171774 tid 3 2014-07-01 16:55:06,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: snap_create 2014-07-01 16:55:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 310 (ObjNum 448 snap 0 seq_num 448) dirty exists 2014-07-01 16:55:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:913: expect (ObjNum 43 snap 0 seq_num 43) 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 1 to mira0437250-302 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 2 to mira0437250-302 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 3 to mira0437250-302 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 5 to mira0437250-302 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: finishing write tid 6 to mira0437250-302 2014-07-01 16:55:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 195 (ObjNum 653 snap 71 seq_num 653) dirty exists 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: finishing write tid 1 to mira0435601-431 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: finishing write tid 2 to mira0435601-431 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: finishing write tid 3 to mira0435601-431 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: finishing write tid 5 to mira0435601-431 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: finishing write tid 6 to mira0435601-431 2014-07-01 16:55:06,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 254 (ObjNum 1674 snap 0 seq_num 1674) dirty exists 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: finishing write tid 4 to mira0435601-367 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 238 (ObjNum 1675 snap 0 seq_num 1675) dirty exists 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 1 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 2 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 3 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 4 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 6 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: finishing write tid 7 to mira0435601-267 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 259 (ObjNum 1677 snap 0 seq_num 1677) dirty exists 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2634: done (4 left) 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: done (3 left) 2014-07-01 16:55:06,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: done (2 left) 2014-07-01 16:55:06,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: done (1 left) 2014-07-01 16:55:06,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2639: done (0 left) 2014-07-01 16:55:06,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2640: read oid 316 snap -1 2014-07-01 16:55:06,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: read oid 164 snap -1 2014-07-01 16:55:06,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2642: read oid 298 snap -1 2014-07-01 16:55:06,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 16:55:06,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: seq_num 1679 ranges {2052096=770048} 2014-07-01 16:55:06,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: writing mira0435601-336 from 2052096 to 2822144 tid 1 2014-07-01 16:55:06,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: read oid 273 snap -1 2014-07-01 16:55:06,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2645: read oid 359 snap -1 2014-07-01 16:55:06,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: read oid 20 snap -1 2014-07-01 16:55:06,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2647: read oid 479 snap -1 2014-07-01 16:55:06,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: read oid 97 snap -1 2014-07-01 16:55:06,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 476 current snap is 0 2014-07-01 16:55:06,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: seq_num 1680 ranges {0=643072} 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: writing mira0435601-476 from 0 to 643072 tid 1 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2650: read oid 246 snap -1 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 395 current snap is 0 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: seq_num 1681 ranges {0=516096} 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: writing mira0435601-395 from 0 to 516096 tid 1 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2652: read oid 18 snap -1 2014-07-01 16:55:07,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: expect (ObjNum 1663 snap 0 seq_num 1663) 2014-07-01 16:55:07,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:912: done (4 left) 2014-07-01 16:55:07,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:913: done (3 left) 2014-07-01 16:55:07,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:915: done (2 left) 2014-07-01 16:55:07,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:916: done (1 left) 2014-07-01 16:55:07,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: snap_create 2014-07-01 16:55:08,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2640: expect (ObjNum 1345 snap 0 seq_num 1345) 2014-07-01 16:55:08,554.554 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:55:08,554.554 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:55:08,554.554 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 2 to 0.147840706842 2014-07-01 16:55:08,555.555 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 0.147840706842' 2014-07-01 16:55:08,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:917: done (1 left) 2014-07-01 16:55:08,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: delete oid 245 current snap is 73 2014-07-01 16:55:09,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: expect (ObjNum 1400 snap 0 seq_num 1400) 2014-07-01 16:55:09,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 16:55:09,912.912 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 0.147841 (896882) 2014-07-01 16:55:10,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:914: done (1 left) 2014-07-01 16:55:10,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:918: done (0 left) 2014-07-01 16:55:10,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: read oid 163 snap -1 2014-07-01 16:55:10,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: snap_remove snap 31 2014-07-01 16:55:10,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: expect (ObjNum 536 snap 14 seq_num 536) 2014-07-01 16:55:10,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:919: done (1 left) 2014-07-01 16:55:10,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:920: done (0 left) 2014-07-01 16:55:10,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: write oid 224 current snap is 73 2014-07-01 16:55:10,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: seq_num 655 ranges {128184=1} 2014-07-01 16:55:10,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: writing mira0437250-224 from 128184 to 128185 tid 1 2014-07-01 16:55:10,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:922: rollback oid 402 current snap is 73 2014-07-01 16:55:10,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 402 to 34 2014-07-01 16:55:10,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: rollback oid 129 current snap is 73 2014-07-01 16:55:10,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 129 to 71 2014-07-01 16:55:10,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: write oid 239 current snap is 73 2014-07-01 16:55:10,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: seq_num 656 ranges {471991=400000} 2014-07-01 16:55:10,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: writing mira0437250-239 from 471991 to 871991 tid 1 2014-07-01 16:55:10,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: write oid 19 current snap is 73 2014-07-01 16:55:10,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: seq_num 657 ranges {416821=679580,1525484=91337} 2014-07-01 16:55:10,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: writing mira0437250-19 from 416821 to 1096401 tid 1 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: writing mira0437250-19 from 1525484 to 1616821 tid 2 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: read oid 151 snap -1 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:927: snap_create 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: finishing write tid 1 to mira0437250-19 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: finishing write tid 2 to mira0437250-19 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: finishing write tid 4 to mira0437250-19 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: finishing write tid 5 to mira0437250-19 2014-07-01 16:55:11,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 262 (ObjNum 657 snap 73 seq_num 657) dirty exists 2014-07-01 16:55:11,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 294 (ObjNum 401 snap 0 seq_num 401) dirty exists 2014-07-01 16:55:11,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:2642: expect (ObjNum 1302 snap 0 seq_num 1302) 2014-07-01 16:55:11,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 1 to mira0437250-239 2014-07-01 16:55:11,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 3 to mira0437250-239 2014-07-01 16:55:11,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: finishing write tid 4 to mira0437250-239 2014-07-01 16:55:11,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 353 (ObjNum 656 snap 73 seq_num 656) dirty exists 2014-07-01 16:55:11,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: expect (ObjNum 150 snap 0 seq_num 150) 2014-07-01 16:55:12,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: finishing write tid 1 to mira0437250-224 2014-07-01 16:55:12,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: finishing write tid 3 to mira0437250-224 2014-07-01 16:55:12,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: finishing write tid 4 to mira0437250-224 2014-07-01 16:55:12,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 170 (ObjNum 655 snap 73 seq_num 655) dirty exists 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 172 (ObjNum 128 snap 0 seq_num 128) dirty exists 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:921: done (6 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:922: done (5 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:923: done (4 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:924: done (3 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:925: done (2 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:926: done (1 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:927: done (0 left) 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: write oid 56 current snap is 74 2014-07-01 16:55:12,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: seq_num 658 ranges {512051=403833,1312050=1} 2014-07-01 16:55:12,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0437250-56 from 512051 to 915884 tid 1 2014-07-01 16:55:12,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: writing mira0437250-56 from 1312050 to 1312051 tid 2 2014-07-01 16:55:12,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: delete oid 379 current snap is 74 2014-07-01 16:55:14,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2645: expect (ObjNum 1542 snap 0 seq_num 1542) 2014-07-01 16:55:14,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 1 to mira0437250-56 2014-07-01 16:55:14,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 2 to mira0437250-56 2014-07-01 16:55:14,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 4 to mira0437250-56 2014-07-01 16:55:14,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:929: done (1 left) 2014-07-01 16:55:14,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:930: snap_remove snap 20 2014-07-01 16:55:14,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: finishing write tid 5 to mira0437250-56 2014-07-01 16:55:14,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 320 (ObjNum 658 snap 74 seq_num 658) dirty exists 2014-07-01 16:55:14,913.913 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:55:14,913.913 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:55:15,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:928: done (1 left) 2014-07-01 16:55:15,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:930: done (0 left) 2014-07-01 16:55:15,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: read oid 265 snap -1 2014-07-01 16:55:15,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:932: rollback oid 371 current snap is 74 2014-07-01 16:55:15,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 371 to 59 2014-07-01 16:55:15,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:933: delete oid 286 current snap is 74 2014-07-01 16:55:15,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:933: done (2 left) 2014-07-01 16:55:15,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: write oid 350 current snap is 74 2014-07-01 16:55:15,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: seq_num 661 ranges {613292=755342,1945917=782808,3013291=1} 2014-07-01 16:55:15,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: writing mira0437250-350 from 613292 to 1368634 tid 1 2014-07-01 16:55:15,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: writing mira0437250-350 from 1945917 to 2728725 tid 2 2014-07-01 16:55:15,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: writing mira0437250-350 from 3013291 to 3013292 tid 3 2014-07-01 16:55:15,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: rollback oid 331 current snap is 74 2014-07-01 16:55:15,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 331 to 67 2014-07-01 16:55:15,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: expect (ObjNum 264 snap 0 seq_num 264) 2014-07-01 16:55:15,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 245 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 216 (ObjNum 330 snap 0 seq_num 330) dirty exists 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: finishing write tid 1 to mira0437250-350 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: finishing write tid 2 to mira0437250-350 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: finishing write tid 3 to mira0437250-350 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: finishing write tid 5 to mira0437250-350 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: finishing write tid 6 to mira0437250-350 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 244 (ObjNum 661 snap 74 seq_num 661) dirty exists 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:931: done (3 left) 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:932: done (2 left) 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:934: done (1 left) 2014-07-01 16:55:15,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:935: done (0 left) 2014-07-01 16:55:15,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: write oid 254 current snap is 74 2014-07-01 16:55:15,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: seq_num 662 ranges {708522=478570,1776401=604427,2708521=1} 2014-07-01 16:55:15,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: writing mira0437250-254 from 708522 to 1187092 tid 1 2014-07-01 16:55:15,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: writing mira0437250-254 from 1776401 to 2380828 tid 2 2014-07-01 16:55:15,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: writing mira0437250-254 from 2708521 to 2708522 tid 3 2014-07-01 16:55:15,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:937: rollback oid 267 current snap is 74 2014-07-01 16:55:15,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 267 to 57 2014-07-01 16:55:15,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 1 to mira0437250-254 2014-07-01 16:55:15,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:938: snap_remove snap 63 2014-07-01 16:55:15,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 2 to mira0437250-254 2014-07-01 16:55:15,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 3 to mira0437250-254 2014-07-01 16:55:15,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 5 to mira0437250-254 2014-07-01 16:55:15,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: finishing write tid 6 to mira0437250-254 2014-07-01 16:55:15,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 276 (ObjNum 662 snap 74 seq_num 662) dirty exists 2014-07-01 16:55:16,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:936: done (2 left) 2014-07-01 16:55:16,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:938: done (1 left) 2014-07-01 16:55:16,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: snap_create 2014-07-01 16:55:16,172.172 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:55:16,180.180 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:55:17,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:939: done (1 left) 2014-07-01 16:55:17,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: read oid 411 snap -1 2014-07-01 16:55:17,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: delete oid 211 current snap is 75 2014-07-01 16:55:17,187.187 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:55:17,199.199 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:55:17,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:941: done (2 left) 2014-07-01 16:55:17,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: write oid 51 current snap is 75 2014-07-01 16:55:17,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: seq_num 664 ranges {713508=1} 2014-07-01 16:55:17,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: writing mira0437250-51 from 713508 to 713509 tid 1 2014-07-01 16:55:17,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: write oid 134 current snap is 75 2014-07-01 16:55:17,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: seq_num 665 ranges {409762=1} 2014-07-01 16:55:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: writing mira0437250-134 from 409762 to 409763 tid 1 2014-07-01 16:55:17,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: write oid 252 current snap is 75 2014-07-01 16:55:17,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: seq_num 666 ranges {602145=401883,1742561=459584} 2014-07-01 16:55:17,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: writing mira0437250-252 from 602145 to 1004028 tid 1 2014-07-01 16:55:17,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: writing mira0437250-252 from 1742561 to 2202145 tid 2 2014-07-01 16:55:17,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: write oid 344 current snap is 75 2014-07-01 16:55:17,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: seq_num 667 ranges {698398=773688,1898397=1} 2014-07-01 16:55:17,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: writing mira0437250-344 from 698398 to 1472086 tid 1 2014-07-01 16:55:17,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: writing mira0437250-344 from 1898397 to 1898398 tid 2 2014-07-01 16:55:17,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: write oid 268 current snap is 75 2014-07-01 16:55:17,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: seq_num 668 ranges {511004=781588,2088201=22803} 2014-07-01 16:55:17,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: writing mira0437250-268 from 511004 to 1292592 tid 1 2014-07-01 16:55:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: writing mira0437250-268 from 2088201 to 2111004 tid 2 2014-07-01 16:55:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: write oid 492 current snap is 75 2014-07-01 16:55:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: seq_num 669 ranges {703386=753393,1862367=482289,2807311=429255,3865129=38257} 2014-07-01 16:55:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: writing mira0437250-492 from 703386 to 1456779 tid 1 2014-07-01 16:55:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: writing mira0437250-492 from 1862367 to 2344656 tid 2 2014-07-01 16:55:17,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: writing mira0437250-492 from 2807311 to 3236566 tid 3 2014-07-01 16:55:17,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: writing mira0437250-492 from 3865129 to 3903386 tid 4 2014-07-01 16:55:17,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: write oid 287 current snap is 75 2014-07-01 16:55:17,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 1 to mira0437250-51 2014-07-01 16:55:17,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 3 to mira0437250-51 2014-07-01 16:55:17,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: finishing write tid 4 to mira0437250-51 2014-07-01 16:55:17,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 51 v 324 (ObjNum 664 snap 75 seq_num 664) dirty exists 2014-07-01 16:55:17,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: finishing write tid 1 to mira0437250-252 2014-07-01 16:55:17,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: seq_num 670 ranges {798616=476621,1726899=703908,3159960=438656} 2014-07-01 16:55:17,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: writing mira0437250-287 from 798616 to 1275237 tid 1 2014-07-01 16:55:17,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: writing mira0437250-287 from 1726899 to 2430807 tid 2 2014-07-01 16:55:17,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: writing mira0437250-287 from 3159960 to 3598616 tid 3 2014-07-01 16:55:17,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:942: done (8 left) 2014-07-01 16:55:17,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:949: snap_create 2014-07-01 16:55:17,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: finishing write tid 2 to mira0437250-252 2014-07-01 16:55:17,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: finishing write tid 4 to mira0437250-252 2014-07-01 16:55:17,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: finishing write tid 5 to mira0437250-252 2014-07-01 16:55:17,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 261 (ObjNum 666 snap 75 seq_num 666) dirty exists 2014-07-01 16:55:17,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 235 (ObjNum 266 snap 0 seq_num 266) dirty exists 2014-07-01 16:55:17,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 1 to mira0437250-268 2014-07-01 16:55:17,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 1 to mira0437250-287 2014-07-01 16:55:17,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 2 to mira0437250-268 2014-07-01 16:55:17,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 4 to mira0437250-268 2014-07-01 16:55:17,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 2 to mira0437250-287 2014-07-01 16:55:17,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: finishing write tid 5 to mira0437250-268 2014-07-01 16:55:17,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 159 (ObjNum 668 snap 75 seq_num 668) dirty exists 2014-07-01 16:55:17,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: expect (ObjNum 410 snap 0 seq_num 410) 2014-07-01 16:55:18,164.164 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:55:18,178.178 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:55:18,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: expect (ObjNum 540 snap 0 seq_num 540) 2014-07-01 16:55:19,216.216 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:55:19,230.230 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:55:19,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 3 to mira0437250-287 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 5 to mira0437250-287 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 1 to mira0437250-344 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 2 to mira0437250-344 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 4 to mira0437250-344 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: finishing write tid 6 to mira0437250-287 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 242 (ObjNum 670 snap 75 seq_num 670) dirty exists 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: finishing write tid 5 to mira0437250-344 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 239 (ObjNum 667 snap 75 seq_num 667) dirty exists 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 1 to mira0437250-134 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 3 to mira0437250-134 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: finishing write tid 4 to mira0437250-134 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 196 (ObjNum 665 snap 75 seq_num 665) dirty exists 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 1 to mira0437250-492 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 2 to mira0437250-492 2014-07-01 16:55:19,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 3 to mira0437250-492 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 4 to mira0437250-492 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 6 to mira0437250-492 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: finishing write tid 7 to mira0437250-492 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 185 (ObjNum 669 snap 75 seq_num 669) dirty exists 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:937: done (8 left) 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:940: done (7 left) 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:943: done (6 left) 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:944: done (5 left) 2014-07-01 16:55:19,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:945: done (4 left) 2014-07-01 16:55:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:946: done (3 left) 2014-07-01 16:55:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:947: done (2 left) 2014-07-01 16:55:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:948: done (1 left) 2014-07-01 16:55:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:949: done (0 left) 2014-07-01 16:55:19,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:950: read oid 83 snap -1 2014-07-01 16:55:19,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: delete oid 144 current snap is 76 2014-07-01 16:55:19,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:950: expect (ObjNum 82 snap 0 seq_num 82) 2014-07-01 16:55:19,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:950: done (1 left) 2014-07-01 16:55:19,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:951: done (0 left) 2014-07-01 16:55:19,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:952: rollback oid 112 current snap is 76 2014-07-01 16:55:19,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 112 to 71 2014-07-01 16:55:19,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: snap_create 2014-07-01 16:55:20,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:953: done (1 left) 2014-07-01 16:55:20,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: rollback oid 116 current snap is 77 2014-07-01 16:55:20,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 116 to 58 2014-07-01 16:55:20,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:955: read oid 179 snap -1 2014-07-01 16:55:20,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: snap_remove snap 76 2014-07-01 16:55:20,376.376 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:55:20,389.389 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:55:20,389.389 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:20,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:955: expect (ObjNum 178 snap 0 seq_num 178) 2014-07-01 16:55:20,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:2647: expect (ObjNum 1562 snap 0 seq_num 1562) 2014-07-01 16:55:21,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 263 (ObjNum 115 snap 0 seq_num 115) dirty exists 2014-07-01 16:55:21,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 112 v 165 (ObjNum 111 snap 0 seq_num 111) dirty exists 2014-07-01 16:55:21,182.182 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:21,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2650: expect (ObjNum 857 snap 0 seq_num 857) 2014-07-01 16:55:21,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:952: done (3 left) 2014-07-01 16:55:21,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:954: done (2 left) 2014-07-01 16:55:21,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:955: done (1 left) 2014-07-01 16:55:21,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:956: done (0 left) 2014-07-01 16:55:21,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: read oid 433 snap 65 2014-07-01 16:55:21,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: snap_create 2014-07-01 16:55:21,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: expect (ObjNum 432 snap 0 seq_num 432) 2014-07-01 16:55:22,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: expect (ObjNum 1570 snap 0 seq_num 1570) 2014-07-01 16:55:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:957: done (1 left) 2014-07-01 16:55:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:958: done (0 left) 2014-07-01 16:55:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: write oid 406 current snap is 78 2014-07-01 16:55:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: seq_num 672 ranges {403603=456326,1368667=234936} 2014-07-01 16:55:22,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0437250-406 from 403603 to 859929 tid 1 2014-07-01 16:55:22,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: writing mira0437250-406 from 1368667 to 1603603 tid 2 2014-07-01 16:55:22,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:960: rollback oid 287 current snap is 78 2014-07-01 16:55:22,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 287 to 72 2014-07-01 16:55:22,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: snap_remove snap 77 2014-07-01 16:55:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:961: done (2 left) 2014-07-01 16:55:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:962: snap_remove snap 62 2014-07-01 16:55:24,148.148 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 1 to mira0437250-406 2014-07-01 16:55:24,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 2 to mira0437250-406 2014-07-01 16:55:24,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 4 to mira0437250-406 2014-07-01 16:55:24,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 248 (ObjNum 633 snap 60 seq_num 633) dirty exists 2014-07-01 16:55:24,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: finishing write tid 5 to mira0437250-406 2014-07-01 16:55:24,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 406 v 246 (ObjNum 672 snap 78 seq_num 672) dirty exists 2014-07-01 16:55:24,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:959: done (2 left) 2014-07-01 16:55:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:960: done (1 left) 2014-07-01 16:55:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:962: done (0 left) 2014-07-01 16:55:24,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:963: snap_create 2014-07-01 16:55:24,866.866 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:25,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:963: done (0 left) 2014-07-01 16:55:25,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: write oid 20 current snap is 79 2014-07-01 16:55:25,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: seq_num 673 ranges {499857=428130,1299856=1} 2014-07-01 16:55:25,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: writing mira0437250-20 from 499857 to 927987 tid 1 2014-07-01 16:55:25,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: writing mira0437250-20 from 1299856 to 1299857 tid 2 2014-07-01 16:55:25,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: write oid 245 current snap is 79 2014-07-01 16:55:25,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: seq_num 674 ranges {443662=551358,1712258=331404} 2014-07-01 16:55:25,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0437250-245 from 443662 to 995020 tid 1 2014-07-01 16:55:25,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: writing mira0437250-245 from 1712258 to 2043662 tid 2 2014-07-01 16:55:25,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: read oid 14 snap 49 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: delete oid 279 current snap is 79 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: finishing write tid 1 to mira0437250-20 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: finishing write tid 2 to mira0437250-20 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: finishing write tid 4 to mira0437250-20 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: finishing write tid 5 to mira0437250-20 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 306 (ObjNum 673 snap 79 seq_num 673) dirty exists 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:964: done (3 left) 2014-07-01 16:55:25,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:967: done (2 left) 2014-07-01 16:55:25,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: write oid 59 current snap is 79 2014-07-01 16:55:25,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: seq_num 676 ranges {483722=780663,1922347=561375} 2014-07-01 16:55:25,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: writing mira0437250-59 from 483722 to 1264385 tid 1 2014-07-01 16:55:25,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: writing mira0437250-59 from 1922347 to 2483722 tid 2 2014-07-01 16:55:25,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: read oid 234 snap 59 2014-07-01 16:55:25,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: expect (ObjNum 13 snap 0 seq_num 13) 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: finishing write tid 1 to mira0435601-336 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: finishing write tid 3 to mira0435601-336 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: finishing write tid 1 to mira0435601-395 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: finishing write tid 3 to mira0435601-395 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: finishing write tid 1 to mira0435601-476 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: finishing write tid 3 to mira0435601-476 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: finishing write tid 4 to mira0435601-476 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 240 (ObjNum 1680 snap 0 seq_num 1680) dirty exists 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: finishing write tid 4 to mira0435601-336 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 176 (ObjNum 1679 snap 0 seq_num 1679) dirty exists 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: finishing write tid 4 to mira0435601-395 2014-07-01 16:55:26,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 251 (ObjNum 1681 snap 0 seq_num 1681) dirty exists 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2640: done (12 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: done (11 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2642: done (10 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: done (9 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: done (8 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2645: done (7 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: done (6 left) 2014-07-01 16:55:26,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2647: done (5 left) 2014-07-01 16:55:26,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: done (4 left) 2014-07-01 16:55:26,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: done (3 left) 2014-07-01 16:55:26,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2650: done (2 left) 2014-07-01 16:55:26,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: done (1 left) 2014-07-01 16:55:26,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2652: done (0 left) 2014-07-01 16:55:26,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2653: read oid 36 snap -1 2014-07-01 16:55:26,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 141 current snap is 0 2014-07-01 16:55:26,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: seq_num 1682 ranges {0=188416} 2014-07-01 16:55:26,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: writing mira0435601-141 from 0 to 188416 tid 1 2014-07-01 16:55:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 301 current snap is 0 2014-07-01 16:55:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: seq_num 1683 ranges {1503232=61440} 2014-07-01 16:55:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: writing mira0435601-301 from 1503232 to 1564672 tid 1 2014-07-01 16:55:26,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 413 current snap is 0 2014-07-01 16:55:26,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: seq_num 1684 ranges {1146880=733184,1880064=413696,2293760=622592,2916352=438272,3354624=131072} 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: writing mira0435601-413 from 1146880 to 1880064 tid 1 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: writing mira0435601-413 from 1880064 to 2293760 tid 2 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: writing mira0435601-413 from 2293760 to 2916352 tid 3 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: writing mira0435601-413 from 2916352 to 3354624 tid 4 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: writing mira0435601-413 from 3354624 to 3485696 tid 5 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 219 current snap is 0 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: seq_num 1685 ranges {2932736=761856,3694592=401408} 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: writing mira0435601-219 from 2932736 to 3694592 tid 1 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: writing mira0435601-219 from 3694592 to 4096000 tid 2 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 16:55:26,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: seq_num 1686 ranges {5230592=684032,5914624=581632,6496256=520192,7016448=102400} 2014-07-01 16:55:26,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: writing mira0435601-226 from 5230592 to 5914624 tid 1 2014-07-01 16:55:26,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: writing mira0435601-226 from 5914624 to 6496256 tid 2 2014-07-01 16:55:26,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: writing mira0435601-226 from 6496256 to 7016448 tid 3 2014-07-01 16:55:26,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: writing mira0435601-226 from 7016448 to 7118848 tid 4 2014-07-01 16:55:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 215 current snap is 0 2014-07-01 16:55:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: seq_num 1687 ranges {4198400=552960,4751360=409600,5160960=794624} 2014-07-01 16:55:26,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: writing mira0435601-215 from 4198400 to 4751360 tid 1 2014-07-01 16:55:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: writing mira0435601-215 from 4751360 to 5160960 tid 2 2014-07-01 16:55:26,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: writing mira0435601-215 from 5160960 to 5955584 tid 3 2014-07-01 16:55:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: read oid 479 snap -1 2014-07-01 16:55:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2653: expect (ObjNum 1365 snap 0 seq_num 1365) 2014-07-01 16:55:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: expect (ObjNum 233 snap 0 seq_num 233) 2014-07-01 16:55:27,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:966: done (3 left) 2014-07-01 16:55:27,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:969: done (2 left) 2014-07-01 16:55:27,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: write oid 347 current snap is 79 2014-07-01 16:55:27,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: seq_num 677 ranges {544904=752467,1716735=580747,3046489=642939,3744903=1} 2014-07-01 16:55:27,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: writing mira0437250-347 from 544904 to 1297371 tid 1 2014-07-01 16:55:27,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: writing mira0437250-347 from 1716735 to 2297482 tid 2 2014-07-01 16:55:27,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: writing mira0437250-347 from 3046489 to 3689428 tid 3 2014-07-01 16:55:27,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: writing mira0437250-347 from 3744903 to 3744904 tid 4 2014-07-01 16:55:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: write oid 221 current snap is 79 2014-07-01 16:55:27,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: seq_num 678 ranges {640133=475696,1829843=650942,3247714=192419} 2014-07-01 16:55:27,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: writing mira0437250-221 from 640133 to 1115829 tid 1 2014-07-01 16:55:27,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: writing mira0437250-221 from 1829843 to 2480785 tid 2 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: writing mira0437250-221 from 3247714 to 3440133 tid 3 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: read oid 207 snap -1 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 1 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 2 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 3 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 4 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 6 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: finishing write tid 7 to mira0437250-347 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 229 (ObjNum 677 snap 79 seq_num 677) dirty exists 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 1 to mira0437250-221 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:970: done (4 left) 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:973: read oid 129 snap -1 2014-07-01 16:55:27,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:974: snap_remove snap 57 2014-07-01 16:55:27,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 2 to mira0437250-221 2014-07-01 16:55:27,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 3 to mira0437250-221 2014-07-01 16:55:27,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 5 to mira0437250-221 2014-07-01 16:55:27,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: finishing write tid 6 to mira0437250-221 2014-07-01 16:55:27,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 332 (ObjNum 678 snap 79 seq_num 678) dirty exists 2014-07-01 16:55:27,664.664 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:971: done (5 left) 2014-07-01 16:55:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:974: done (4 left) 2014-07-01 16:55:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: write oid 208 current snap is 79 2014-07-01 16:55:27,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: seq_num 679 ranges {183938=1} 2014-07-01 16:55:27,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: writing mira0437250-208 from 183938 to 183939 tid 1 2014-07-01 16:55:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: write oid 477 current snap is 79 2014-07-01 16:55:27,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: seq_num 680 ranges {528769=400000} 2014-07-01 16:55:27,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: writing mira0437250-477 from 528769 to 928769 tid 1 2014-07-01 16:55:27,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: write oid 166 current snap is 79 2014-07-01 16:55:27,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: seq_num 681 ranges {740351=400000} 2014-07-01 16:55:27,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: writing mira0437250-166 from 740351 to 1140351 tid 1 2014-07-01 16:55:27,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:978: rollback oid 322 current snap is 79 2014-07-01 16:55:27,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 322 to 53 2014-07-01 16:55:27,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: write oid 62 current snap is 79 2014-07-01 16:55:27,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: seq_num 682 ranges {685180=551457,1816227=68953} 2014-07-01 16:55:27,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: writing mira0437250-62 from 685180 to 1236637 tid 1 2014-07-01 16:55:27,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: writing mira0437250-62 from 1816227 to 1885180 tid 2 2014-07-01 16:55:27,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:980: snap_create 2014-07-01 16:55:28,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: finishing write tid 1 to mira0437250-166 2014-07-01 16:55:28,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: finishing write tid 3 to mira0437250-166 2014-07-01 16:55:28,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 235 (ObjNum 565 snap 26 seq_num 565) dirty exists 2014-07-01 16:55:28,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 1 to mira0437250-477 2014-07-01 16:55:28,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 3 to mira0437250-477 2014-07-01 16:55:28,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: finishing write tid 4 to mira0437250-166 2014-07-01 16:55:28,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 285 (ObjNum 681 snap 79 seq_num 681) dirty exists 2014-07-01 16:55:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: finishing write tid 4 to mira0437250-477 2014-07-01 16:55:28,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 276 (ObjNum 680 snap 79 seq_num 680) dirty exists 2014-07-01 16:55:28,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: finishing write tid 1 to mira0437250-59 2014-07-01 16:55:28,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: finishing write tid 2 to mira0437250-59 2014-07-01 16:55:28,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: finishing write tid 4 to mira0437250-59 2014-07-01 16:55:28,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 1 to mira0437250-208 2014-07-01 16:55:28,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 3 to mira0437250-208 2014-07-01 16:55:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:976: done (9 left) 2014-07-01 16:55:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:977: done (8 left) 2014-07-01 16:55:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:978: done (7 left) 2014-07-01 16:55:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:980: done (6 left) 2014-07-01 16:55:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:981: snap_create 2014-07-01 16:55:28,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: finishing write tid 5 to mira0437250-59 2014-07-01 16:55:28,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 243 (ObjNum 676 snap 79 seq_num 676) dirty exists 2014-07-01 16:55:28,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 1 to mira0435601-301 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 3 to mira0435601-301 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 4 to mira0435601-301 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 281 (ObjNum 1683 snap 0 seq_num 1683) dirty exists 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 1 to mira0435601-413 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 2 to mira0435601-413 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2653: done (7 left) 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: done (6 left) 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 331 current snap is 0 2014-07-01 16:55:28,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: seq_num 1688 ranges {0=581632} 2014-07-01 16:55:28,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: writing mira0435601-331 from 0 to 581632 tid 1 2014-07-01 16:55:28,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 322 current snap is 0 2014-07-01 16:55:28,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: seq_num 1689 ranges {454656=454656} 2014-07-01 16:55:28,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: writing mira0435601-322 from 454656 to 909312 tid 1 2014-07-01 16:55:28,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2663: read oid 432 snap -1 2014-07-01 16:55:28,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 3 to mira0435601-413 2014-07-01 16:55:28,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 4 to mira0435601-413 2014-07-01 16:55:28,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 5 to mira0435601-413 2014-07-01 16:55:28,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 7 to mira0435601-413 2014-07-01 16:55:28,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: finishing write tid 8 to mira0435601-413 2014-07-01 16:55:28,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 265 (ObjNum 1684 snap 0 seq_num 1684) dirty exists 2014-07-01 16:55:28,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 1 to mira0435601-226 2014-07-01 16:55:28,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 2 to mira0435601-226 2014-07-01 16:55:28,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 3 to mira0435601-226 2014-07-01 16:55:28,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 4 to mira0435601-226 2014-07-01 16:55:28,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 6 to mira0435601-226 2014-07-01 16:55:28,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: done (8 left) 2014-07-01 16:55:28,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 206 current snap is 0 2014-07-01 16:55:29,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: seq_num 1690 ranges {1998848=479232,2478080=561152,3039232=413696,3452928=229376} 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: writing mira0435601-206 from 1998848 to 2478080 tid 1 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: writing mira0435601-206 from 2478080 to 3039232 tid 2 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: writing mira0435601-206 from 3039232 to 3452928 tid 3 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: writing mira0435601-206 from 3452928 to 3682304 tid 4 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 481 current snap is 0 2014-07-01 16:55:29,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: seq_num 1691 ranges {2555904=647168,3203072=401408} 2014-07-01 16:55:29,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: writing mira0435601-481 from 2555904 to 3203072 tid 1 2014-07-01 16:55:29,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: writing mira0435601-481 from 3203072 to 3604480 tid 2 2014-07-01 16:55:29,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2666: read oid 124 snap -1 2014-07-01 16:55:29,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: finishing write tid 7 to mira0435601-226 2014-07-01 16:55:29,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 132 (ObjNum 1686 snap 0 seq_num 1686) dirty exists 2014-07-01 16:55:29,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2663: expect (ObjNum 1407 snap 0 seq_num 1407) 2014-07-01 16:55:29,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: finishing write tid 4 to mira0437250-208 2014-07-01 16:55:29,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 255 (ObjNum 679 snap 79 seq_num 679) dirty exists 2014-07-01 16:55:30,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:968: done (6 left) 2014-07-01 16:55:30,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:975: done (5 left) 2014-07-01 16:55:30,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:981: done (4 left) 2014-07-01 16:55:30,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: write oid 74 current snap is 81 2014-07-01 16:55:30,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: seq_num 683 ranges {630010=523261,1778935=504766,2630009=1} 2014-07-01 16:55:30,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: writing mira0437250-74 from 630010 to 1153271 tid 1 2014-07-01 16:55:30,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: writing mira0437250-74 from 1778935 to 2283701 tid 2 2014-07-01 16:55:30,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: writing mira0437250-74 from 2630009 to 2630010 tid 3 2014-07-01 16:55:30,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:983: snap_create 2014-07-01 16:55:30,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: expect (ObjNum 206 snap 0 seq_num 206) 2014-07-01 16:55:30,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: finishing write tid 1 to mira0437250-62 2014-07-01 16:55:30,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: finishing write tid 2 to mira0437250-62 2014-07-01 16:55:30,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: finishing write tid 4 to mira0437250-62 2014-07-01 16:55:30,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: finishing write tid 5 to mira0437250-62 2014-07-01 16:55:30,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 367 (ObjNum 682 snap 79 seq_num 682) dirty exists 2014-07-01 16:55:31,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:972: done (5 left) 2014-07-01 16:55:31,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:979: done (4 left) 2014-07-01 16:55:31,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:983: done (3 left) 2014-07-01 16:55:31,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: snap_create 2014-07-01 16:55:31,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 1 to mira0437250-245 2014-07-01 16:55:31,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 2 to mira0437250-245 2014-07-01 16:55:31,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 4 to mira0437250-245 2014-07-01 16:55:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: finishing write tid 5 to mira0437250-245 2014-07-01 16:55:31,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 245 v 169 (ObjNum 674 snap 79 seq_num 674) dirty exists 2014-07-01 16:55:31,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:973: expect (ObjNum 128 snap 0 seq_num 128) 2014-07-01 16:55:31,339.339 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:54:44.723464', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881996376, u'health': u'HEALTH_OK', u'kb_used': 14784676, u'store_stats': {u'bytes_total': 8636701, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4442381}}, {u'last_updated': u'2014-07-01 07:54:54.541011', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886069900, u'health': u'HEALTH_OK', u'kb_used': 10711152, u'store_stats': {u'bytes_total': 12830622, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4441998}}, {u'last_updated': u'2014-07-01 07:55:11.323787', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886021776, u'health': u'HEALTH_OK', u'kb_used': 10759276, u'store_stats': {u'bytes_total': 5108638, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4584334}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'22 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 624/2941 objects degraded (21.217%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 7947953, u'degraded_objects': 624, u'num_pgs': 434, u'recovering_keys_per_sec': 0, u'data_bytes': 2374845387, u'degraded_total': 2941, u'bytes_used': 5248221184, u'recovering_objects_per_sec': 2, u'op_per_sec': 0, u'version': 1926, u'pgs_by_state': [{u'count': 22, u'state_name': u'active+recovery_wait'}, {u'count': 391, u'state_name': u'active+clean'}, {u'count': 21, u'state_name': u'active+recovering'}], u'read_bytes_sec': 706965, u'degraded_ratio': u'21.217', u'bytes_avail': 4993334317056}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 778, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:55:31,340.340 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:32,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: finishing write tid 1 to mira0437250-74 2014-07-01 16:55:32,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: finishing write tid 2 to mira0437250-74 2014-07-01 16:55:32,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: finishing write tid 3 to mira0437250-74 2014-07-01 16:55:32,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: finishing write tid 5 to mira0437250-74 2014-07-01 16:55:32,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: finishing write tid 6 to mira0437250-74 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 291 (ObjNum 683 snap 81 seq_num 683) dirty exists 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:965: done (3 left) 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:973: done (2 left) 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:982: done (1 left) 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:984: done (0 left) 2014-07-01 16:55:32,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: snap_remove snap 66 2014-07-01 16:55:32,123.123 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:55:32,123.123 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:32,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 1 to mira0435601-322 2014-07-01 16:55:32,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 3 to mira0435601-322 2014-07-01 16:55:32,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2666: expect (ObjNum 1370 snap 0 seq_num 1370) 2014-07-01 16:55:33,085.085 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 4 to mira0435601-322 2014-07-01 16:55:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 269 (ObjNum 1689 snap 0 seq_num 1689) dirty exists 2014-07-01 16:55:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: expect (ObjNum 1562 snap 0 seq_num 1562) 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: finishing write tid 1 to mira0435601-481 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: finishing write tid 2 to mira0435601-481 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: finishing write tid 4 to mira0435601-481 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: finishing write tid 5 to mira0435601-481 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 147 (ObjNum 1691 snap 0 seq_num 1691) dirty exists 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: finishing write tid 1 to mira0435601-141 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: finishing write tid 3 to mira0435601-141 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: finishing write tid 1 to mira0435601-331 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: finishing write tid 3 to mira0435601-331 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: finishing write tid 4 to mira0435601-331 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 266 (ObjNum 1688 snap 0 seq_num 1688) dirty exists 2014-07-01 16:55:34,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: finishing write tid 4 to mira0435601-141 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 264 (ObjNum 1682 snap 0 seq_num 1682) dirty exists 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 1 to mira0435601-206 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 2 to mira0435601-206 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 3 to mira0435601-206 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 4 to mira0435601-206 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 6 to mira0435601-206 2014-07-01 16:55:34,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: finishing write tid 7 to mira0435601-206 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 221 (ObjNum 1690 snap 0 seq_num 1690) dirty exists 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: done (10 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: done (9 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: done (8 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: done (7 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: done (6 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2663: done (5 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: done (4 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: done (3 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2666: done (2 left) 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 230 current snap is 0 2014-07-01 16:55:34,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: seq_num 1692 ranges {2916352=675840,3592192=622592,4214784=761856,4976640=221184} 2014-07-01 16:55:34,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: writing mira0435601-230 from 2916352 to 3592192 tid 1 2014-07-01 16:55:34,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: writing mira0435601-230 from 3592192 to 4214784 tid 2 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: writing mira0435601-230 from 4214784 to 4976640 tid 3 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: writing mira0435601-230 from 4976640 to 5197824 tid 4 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2668: read oid 191 snap -1 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 1 to mira0435601-230 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 2 to mira0435601-230 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: seq_num 1693 ranges {7385088=700416,8085504=401408} 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: writing mira0435601-391 from 7385088 to 8085504 tid 1 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: writing mira0435601-391 from 8085504 to 8486912 tid 2 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2670: read oid 383 snap -1 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 3 to mira0435601-230 2014-07-01 16:55:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 4 to mira0435601-230 2014-07-01 16:55:34,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 455 current snap is 0 2014-07-01 16:55:34,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: seq_num 1694 ranges {2875392=417792,3293184=786432,4079616=462848,4542464=356352} 2014-07-01 16:55:34,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: writing mira0435601-455 from 2875392 to 3293184 tid 1 2014-07-01 16:55:34,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: writing mira0435601-455 from 3293184 to 4079616 tid 2 2014-07-01 16:55:34,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: writing mira0435601-455 from 4079616 to 4542464 tid 3 2014-07-01 16:55:34,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: writing mira0435601-455 from 4542464 to 4898816 tid 4 2014-07-01 16:55:34,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 16:55:34,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: seq_num 1695 ranges {2465792=495616,2961408=614400,3575808=532480,4108288=57344} 2014-07-01 16:55:34,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: writing mira0435601-327 from 2465792 to 2961408 tid 1 2014-07-01 16:55:34,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: writing mira0435601-327 from 2961408 to 3575808 tid 2 2014-07-01 16:55:34,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: writing mira0435601-327 from 3575808 to 4108288 tid 3 2014-07-01 16:55:34,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: writing mira0435601-327 from 4108288 to 4165632 tid 4 2014-07-01 16:55:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2673: delete oid 207 current snap is 0 2014-07-01 16:55:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 6 to mira0435601-230 2014-07-01 16:55:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: finishing write tid 7 to mira0435601-230 2014-07-01 16:55:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 270 (ObjNum 1692 snap 0 seq_num 1692) dirty exists 2014-07-01 16:55:34,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2668: expect (ObjNum 1153 snap 0 seq_num 1153) 2014-07-01 16:55:36,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:985: done (0 left) 2014-07-01 16:55:36,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: rollback oid 53 current snap is 83 2014-07-01 16:55:36,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 53 to 67 2014-07-01 16:55:36,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: rollback oid 81 current snap is 83 2014-07-01 16:55:36,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 81 to 58 2014-07-01 16:55:36,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: write oid 421 current snap is 83 2014-07-01 16:55:36,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: seq_num 684 ranges {725240=530138,1891020=434220} 2014-07-01 16:55:36,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: writing mira0437250-421 from 725240 to 1255378 tid 1 2014-07-01 16:55:36,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: writing mira0437250-421 from 1891020 to 2325240 tid 2 2014-07-01 16:55:36,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: write oid 283 current snap is 83 2014-07-01 16:55:36,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: seq_num 685 ranges {670070=501942,1853728=528804,3070069=1} 2014-07-01 16:55:36,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: writing mira0437250-283 from 670070 to 1172012 tid 1 2014-07-01 16:55:36,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: writing mira0437250-283 from 1853728 to 2382532 tid 2 2014-07-01 16:55:36,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: writing mira0437250-283 from 3070069 to 3070070 tid 3 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: read oid 217 snap -1 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 81 v 0 (ObjNum 80 snap 0 seq_num 4294967295) dirty dne 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 0 (ObjNum 29 snap 0 seq_num 3) dirty dne 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: finishing write tid 1 to mira0437250-421 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:986: done (4 left) 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:987: done (3 left) 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: read oid 115 snap -1 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: finishing write tid 2 to mira0437250-421 2014-07-01 16:55:36,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: finishing write tid 4 to mira0437250-421 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: finishing write tid 1 to mira0437250-283 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: finishing write tid 5 to mira0437250-421 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 346 (ObjNum 684 snap 83 seq_num 684) dirty exists 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:988: done (3 left) 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: write oid 378 current snap is 83 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: seq_num 686 ranges {330226=1} 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: writing mira0437250-378 from 330226 to 330227 tid 1 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:993: rollback oid 320 current snap is 83 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 320 to 34 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: write oid 387 current snap is 83 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: seq_num 687 ranges {25456=1} 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: writing mira0437250-387 from 25456 to 25457 tid 1 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:995: snap_create 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: finishing write tid 2 to mira0437250-283 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: finishing write tid 3 to mira0437250-283 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: finishing write tid 5 to mira0437250-283 2014-07-01 16:55:36,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: expect (ObjNum 114 snap 0 seq_num 114) 2014-07-01 16:55:38,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: expect (ObjNum 638 snap 64 seq_num 638) 2014-07-01 16:55:38,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 1 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 2 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 3 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 4 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 6 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 7 to mira0435601-455 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 275 (ObjNum 1694 snap 0 seq_num 1694) dirty exists 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: finishing write tid 1 to mira0435601-391 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: finishing write tid 2 to mira0435601-391 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: finishing write tid 1 to mira0435601-219 2014-07-01 16:55:38,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: done (8 left) 2014-07-01 16:55:38,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2668: done (7 left) 2014-07-01 16:55:38,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: done (6 left) 2014-07-01 16:55:38,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2673: done (5 left) 2014-07-01 16:55:38,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: read oid 107 snap -1 2014-07-01 16:55:38,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: finishing write tid 4 to mira0435601-391 2014-07-01 16:55:38,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: finishing write tid 2 to mira0435601-219 2014-07-01 16:55:38,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: finishing write tid 4 to mira0435601-219 2014-07-01 16:55:38,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: finishing write tid 5 to mira0435601-391 2014-07-01 16:55:38,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 272 (ObjNum 1693 snap 0 seq_num 1693) dirty exists 2014-07-01 16:55:38,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2670: expect (ObjNum 957 snap 0 seq_num 957) 2014-07-01 16:55:38,986.986 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:39,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 218 (ObjNum 319 snap 0 seq_num 319) dirty exists 2014-07-01 16:55:39,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: finishing write tid 6 to mira0437250-283 2014-07-01 16:55:39,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 317 (ObjNum 685 snap 83 seq_num 685) dirty exists 2014-07-01 16:55:39,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:989: done (6 left) 2014-07-01 16:55:39,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:990: done (5 left) 2014-07-01 16:55:39,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:991: done (4 left) 2014-07-01 16:55:39,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:993: done (3 left) 2014-07-01 16:55:39,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:995: done (2 left) 2014-07-01 16:55:39,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:996: rollback oid 5 current snap is 84 2014-07-01 16:55:39,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 5 to 52 2014-07-01 16:55:39,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: snap_create 2014-07-01 16:55:39,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 220 (ObjNum 4 snap 0 seq_num 4) dirty exists 2014-07-01 16:55:39,574.574 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:40,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 1 to mira0435601-327 2014-07-01 16:55:40,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 2 to mira0435601-327 2014-07-01 16:55:40,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 3 to mira0435601-327 2014-07-01 16:55:40,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 4 to mira0435601-327 2014-07-01 16:55:40,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: finishing write tid 5 to mira0435601-219 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 311 (ObjNum 1685 snap 0 seq_num 1685) dirty exists 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: done (5 left) 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: done (4 left) 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2670: done (3 left) 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2675: read oid 323 snap -1 2014-07-01 16:55:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: read oid 484 snap -1 2014-07-01 16:55:40,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2677: read oid 236 snap -1 2014-07-01 16:55:40,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 16:55:40,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: seq_num 1697 ranges {6373376=638976,7012352=782336,7794688=20480} 2014-07-01 16:55:40,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: writing mira0435601-21 from 6373376 to 7012352 tid 1 2014-07-01 16:55:40,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: writing mira0435601-21 from 7012352 to 7794688 tid 2 2014-07-01 16:55:40,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: writing mira0435601-21 from 7794688 to 7815168 tid 3 2014-07-01 16:55:40,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 16:55:40,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: expect (ObjNum 1639 snap 0 seq_num 1639) 2014-07-01 16:55:40,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:996: done (3 left) 2014-07-01 16:55:40,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:997: done (2 left) 2014-07-01 16:55:40,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:998: snap_remove snap 81 2014-07-01 16:55:40,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 1 to mira0437250-378 2014-07-01 16:55:40,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 3 to mira0437250-378 2014-07-01 16:55:42,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:998: done (2 left) 2014-07-01 16:55:42,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:999: delete oid 97 current snap is 85 2014-07-01 16:55:42,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: finishing write tid 4 to mira0437250-378 2014-07-01 16:55:42,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 378 v 243 (ObjNum 686 snap 83 seq_num 686) dirty exists 2014-07-01 16:55:42,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 1 to mira0437250-387 2014-07-01 16:55:42,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 3 to mira0437250-387 2014-07-01 16:55:42,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: finishing write tid 4 to mira0437250-387 2014-07-01 16:55:42,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 370 (ObjNum 687 snap 83 seq_num 687) dirty exists 2014-07-01 16:55:42,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:992: done (2 left) 2014-07-01 16:55:42,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:994: done (1 left) 2014-07-01 16:55:42,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:999: done (0 left) 2014-07-01 16:55:42,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: read oid 145 snap 51 2014-07-01 16:55:42,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: read oid 60 snap -1 2014-07-01 16:55:42,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: write oid 81 current snap is 85 2014-07-01 16:55:42,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: seq_num 689 ranges {714093=426279,1514092=1} 2014-07-01 16:55:42,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: writing mira0437250-81 from 714093 to 1140372 tid 1 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: writing mira0437250-81 from 1514092 to 1514093 tid 2 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: rollback oid 308 current snap is 85 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 308 to 35 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: read oid 12 snap 65 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: write oid 41 current snap is 85 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: seq_num 690 ranges {526698=549508,1518149=208549} 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: writing mira0437250-41 from 526698 to 1076206 tid 1 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: writing mira0437250-41 from 1518149 to 1726698 tid 2 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: read oid 215 snap -1 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 0 (ObjNum 1 snap 1 seq_num 16847280) dirty dne 2014-07-01 16:55:42,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: expect (ObjNum 11 snap 0 seq_num 11) 2014-07-01 16:55:42,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: expect (ObjNum 577 snap 28 seq_num 577) 2014-07-01 16:55:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 6 to mira0435601-327 2014-07-01 16:55:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: finishing write tid 1 to mira0435601-215 2014-07-01 16:55:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: finishing write tid 2 to mira0435601-215 2014-07-01 16:55:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: finishing write tid 3 to mira0435601-215 2014-07-01 16:55:42,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: finishing write tid 5 to mira0435601-215 2014-07-01 16:55:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: finishing write tid 6 to mira0435601-215 2014-07-01 16:55:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 278 (ObjNum 1687 snap 0 seq_num 1687) dirty exists 2014-07-01 16:55:42,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2677: expect (ObjNum 941 snap 0 seq_num 941) 2014-07-01 16:55:42,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: expect (ObjNum 59 snap 0 seq_num 59) 2014-07-01 16:55:42,696.696 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:42,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: expect (ObjNum 1632 snap 0 seq_num 1632) 2014-07-01 16:55:42,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: expect (ObjNum 144 snap 0 seq_num 144) 2014-07-01 16:55:43,404.404 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:54:44.723464', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881996376, u'health': u'HEALTH_OK', u'kb_used': 14784676, u'store_stats': {u'bytes_total': 8636701, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4442381}}, {u'last_updated': u'2014-07-01 07:54:54.541011', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886069900, u'health': u'HEALTH_OK', u'kb_used': 10711152, u'store_stats': {u'bytes_total': 12830622, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4441998}}, {u'last_updated': u'2014-07-01 07:55:11.323787', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886021776, u'health': u'HEALTH_OK', u'kb_used': 10759276, u'store_stats': {u'bytes_total': 5108638, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4584334}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'20 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 457/2959 objects degraded (15.444%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 3961606, u'degraded_objects': 457, u'num_pgs': 434, u'recovering_keys_per_sec': 0, u'data_bytes': 2386813198, u'degraded_total': 2959, u'bytes_used': 5310558208, u'recovering_objects_per_sec': 1, u'write_bytes_sec': 588, u'op_per_sec': 2, u'version': 1935, u'pgs_by_state': [{u'count': 11, u'state_name': u'active+recovery_wait'}, {u'count': 403, u'state_name': u'active+clean'}, {u'count': 20, u'state_name': u'active+recovering'}], u'degraded_ratio': u'15.444', u'bytes_avail': 4993271980032}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 783, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:55:43,404.404 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 1 to mira0437250-81 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 2 to mira0437250-81 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 4 to mira0437250-81 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: finishing write tid 5 to mira0437250-81 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 81 v 351 (ObjNum 689 snap 85 seq_num 689) dirty exists 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: finishing write tid 1 to mira0437250-41 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: finishing write tid 2 to mira0437250-41 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: finishing write tid 4 to mira0437250-41 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: finishing write tid 5 to mira0437250-41 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 200 (ObjNum 690 snap 85 seq_num 690) dirty exists 2014-07-01 16:55:43,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1000: done (6 left) 2014-07-01 16:55:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1001: done (5 left) 2014-07-01 16:55:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1002: done (4 left) 2014-07-01 16:55:43,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:1003: done (3 left) 2014-07-01 16:55:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1004: done (2 left) 2014-07-01 16:55:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1005: done (1 left) 2014-07-01 16:55:43,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1006: done (0 left) 2014-07-01 16:55:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: rollback oid 286 current snap is 85 2014-07-01 16:55:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 286 to 72 2014-07-01 16:55:43,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1008: read oid 204 snap -1 2014-07-01 16:55:43,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: snap_create 2014-07-01 16:55:43,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 321 (ObjNum 615 snap 47 seq_num 615) dirty exists 2014-07-01 16:55:44,182.182 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:55:44,183.183 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:44,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: finishing write tid 7 to mira0435601-327 2014-07-01 16:55:44,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 245 (ObjNum 1695 snap 0 seq_num 1695) dirty exists 2014-07-01 16:55:44,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: finishing write tid 1 to mira0435601-21 2014-07-01 16:55:44,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: finishing write tid 2 to mira0435601-21 2014-07-01 16:55:44,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: finishing write tid 3 to mira0435601-21 2014-07-01 16:55:44,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: finishing write tid 5 to mira0435601-21 2014-07-01 16:55:44,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: seq_num 1698 ranges {1695744=266240} 2014-07-01 16:55:44,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: writing mira0435601-211 from 1695744 to 1961984 tid 1 2014-07-01 16:55:44,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: done (7 left) 2014-07-01 16:55:44,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: done (6 left) 2014-07-01 16:55:44,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: done (5 left) 2014-07-01 16:55:44,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2675: done (4 left) 2014-07-01 16:55:44,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: done (3 left) 2014-07-01 16:55:44,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2677: done (2 left) 2014-07-01 16:55:44,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2680: read oid 237 snap -1 2014-07-01 16:55:44,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: finishing write tid 6 to mira0435601-21 2014-07-01 16:55:44,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 249 (ObjNum 1697 snap 0 seq_num 1697) dirty exists 2014-07-01 16:55:44,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: done (2 left) 2014-07-01 16:55:44,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 78 current snap is 0 2014-07-01 16:55:44,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: seq_num 1699 ranges {2633728=692224,3325952=598016,3923968=204800} 2014-07-01 16:55:44,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: writing mira0435601-78 from 2633728 to 3325952 tid 1 2014-07-01 16:55:44,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: writing mira0435601-78 from 3325952 to 3923968 tid 2 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: writing mira0435601-78 from 3923968 to 4128768 tid 3 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2682: read oid 192 snap -1 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2680: done (3 left) 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2683: read oid 364 snap -1 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2684: delete oid 463 current snap is 0 2014-07-01 16:55:44,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2682: expect (ObjNum 1344 snap 0 seq_num 1344) 2014-07-01 16:55:44,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1007: done (2 left) 2014-07-01 16:55:44,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1009: done (1 left) 2014-07-01 16:55:44,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1010: snap_create 2014-07-01 16:55:45,056.056 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:45,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2683: expect (ObjNum 1489 snap 0 seq_num 1489) 2014-07-01 16:55:45,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:1010: done (1 left) 2014-07-01 16:55:45,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: write oid 322 current snap is 87 2014-07-01 16:55:45,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: seq_num 691 ranges {621928=521312,1421927=1} 2014-07-01 16:55:45,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: writing mira0437250-322 from 621928 to 1143240 tid 1 2014-07-01 16:55:45,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: writing mira0437250-322 from 1421927 to 1421928 tid 2 2014-07-01 16:55:45,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1012: snap_create 2014-07-01 16:55:45,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: finishing write tid 1 to mira0437250-322 2014-07-01 16:55:45,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: finishing write tid 2 to mira0437250-322 2014-07-01 16:55:45,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: finishing write tid 4 to mira0437250-322 2014-07-01 16:55:45,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: finishing write tid 5 to mira0437250-322 2014-07-01 16:55:45,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 251 (ObjNum 691 snap 87 seq_num 691) dirty exists 2014-07-01 16:55:46,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1011: done (2 left) 2014-07-01 16:55:46,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1012: done (1 left) 2014-07-01 16:55:46,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: write oid 230 current snap is 88 2014-07-01 16:55:46,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: seq_num 692 ranges {415334=528188,1592941=521995,2887526=327808} 2014-07-01 16:55:46,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: writing mira0437250-230 from 415334 to 943522 tid 1 2014-07-01 16:55:46,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: writing mira0437250-230 from 1592941 to 2114936 tid 2 2014-07-01 16:55:46,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: writing mira0437250-230 from 2887526 to 3215334 tid 3 2014-07-01 16:55:46,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: rollback oid 67 current snap is 88 2014-07-01 16:55:46,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 67 to 73 2014-07-01 16:55:46,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:1015: snap_remove snap 41 2014-07-01 16:55:47,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: finishing write tid 1 to mira0435601-78 2014-07-01 16:55:47,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: finishing write tid 2 to mira0435601-78 2014-07-01 16:55:47,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: finishing write tid 3 to mira0435601-78 2014-07-01 16:55:47,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: finishing write tid 5 to mira0435601-78 2014-07-01 16:55:47,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: finishing write tid 1 to mira0435601-211 2014-07-01 16:55:47,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: finishing write tid 3 to mira0435601-211 2014-07-01 16:55:47,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: finishing write tid 4 to mira0435601-211 2014-07-01 16:55:47,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 149 (ObjNum 1698 snap 0 seq_num 1698) dirty exists 2014-07-01 16:55:50,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 253 (ObjNum 66 snap 0 seq_num 66) dirty exists 2014-07-01 16:55:50,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1014: done (3 left) 2014-07-01 16:55:50,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1015: done (2 left) 2014-07-01 16:55:50,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: write oid 481 current snap is 88 2014-07-01 16:55:50,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: seq_num 693 ranges {511587=501017,1556673=743614,2911586=1} 2014-07-01 16:55:50,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: finishing write tid 6 to mira0435601-78 2014-07-01 16:55:50,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 253 (ObjNum 1699 snap 0 seq_num 1699) dirty exists 2014-07-01 16:55:50,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: writing mira0437250-481 from 511587 to 1012604 tid 1 2014-07-01 16:55:50,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: writing mira0437250-481 from 1556673 to 2300287 tid 2 2014-07-01 16:55:50,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: writing mira0437250-481 from 2911586 to 2911587 tid 3 2014-07-01 16:55:50,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: write oid 487 current snap is 88 2014-07-01 16:55:50,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: seq_num 694 ranges {606817=624245,1937557=448881,2606816=1} 2014-07-01 16:55:50,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: writing mira0437250-487 from 606817 to 1231062 tid 1 2014-07-01 16:55:50,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: writing mira0437250-487 from 1937557 to 2386438 tid 2 2014-07-01 16:55:50,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: writing mira0437250-487 from 2606816 to 2606817 tid 3 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1018: snap_create 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: finishing write tid 1 to mira0437250-487 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: finishing write tid 2 to mira0437250-487 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: finishing write tid 3 to mira0437250-487 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: finishing write tid 5 to mira0437250-487 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: finishing write tid 6 to mira0437250-487 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 330 (ObjNum 694 snap 88 seq_num 694) dirty exists 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 1 to mira0437250-230 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 2 to mira0437250-230 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 3 to mira0437250-230 2014-07-01 16:55:50,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 5 to mira0437250-230 2014-07-01 16:55:50,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: finishing write tid 6 to mira0437250-230 2014-07-01 16:55:50,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 349 (ObjNum 692 snap 88 seq_num 692) dirty exists 2014-07-01 16:55:50,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: finishing write tid 1 to mira0437250-481 2014-07-01 16:55:50,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: finishing write tid 2 to mira0437250-481 2014-07-01 16:55:50,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: finishing write tid 3 to mira0437250-481 2014-07-01 16:55:50,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: finishing write tid 5 to mira0437250-481 2014-07-01 16:55:50,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: finishing write tid 6 to mira0437250-481 2014-07-01 16:55:50,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 256 (ObjNum 693 snap 88 seq_num 693) dirty exists 2014-07-01 16:55:50,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:1008: expect (ObjNum 203 snap 0 seq_num 203) 2014-07-01 16:55:50,783.783 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:50,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: done (4 left) 2014-07-01 16:55:50,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: done (3 left) 2014-07-01 16:55:50,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2682: done (2 left) 2014-07-01 16:55:50,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2683: done (1 left) 2014-07-01 16:55:50,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2684: done (0 left) 2014-07-01 16:55:50,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2685: read oid 194 snap -1 2014-07-01 16:55:50,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2686: delete oid 1 current snap is 0 2014-07-01 16:55:50,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2686: done (1 left) 2014-07-01 16:55:50,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2687: read oid 296 snap -1 2014-07-01 16:55:50,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 111 current snap is 0 2014-07-01 16:55:50,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: seq_num 1702 ranges {1646592=512000,2158592=401408} 2014-07-01 16:55:50,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: writing mira0435601-111 from 1646592 to 2158592 tid 1 2014-07-01 16:55:50,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: writing mira0435601-111 from 2158592 to 2560000 tid 2 2014-07-01 16:55:50,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 264 current snap is 0 2014-07-01 16:55:50,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: seq_num 1703 ranges {802816=786432} 2014-07-01 16:55:50,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: writing mira0435601-264 from 802816 to 1589248 tid 1 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2690: read oid 51 snap -1 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 388 current snap is 0 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: seq_num 1704 ranges {3162112=655360} 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: writing mira0435601-388 from 3162112 to 3817472 tid 1 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2692: read oid 360 snap -1 2014-07-01 16:55:50,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2690: expect (ObjNum 50 snap 0 seq_num 50) 2014-07-01 16:55:51,552.552 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:52,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2685: expect (ObjNum 1592 snap 0 seq_num 1592) 2014-07-01 16:55:52,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:1008: done (4 left) 2014-07-01 16:55:52,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1013: done (3 left) 2014-07-01 16:55:52,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1016: done (2 left) 2014-07-01 16:55:52,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1017: done (1 left) 2014-07-01 16:55:52,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1018: done (0 left) 2014-07-01 16:55:52,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: snap_remove snap 32 2014-07-01 16:55:53,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:1019: done (0 left) 2014-07-01 16:55:53,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: read oid 125 snap -1 2014-07-01 16:55:53,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: read oid 488 snap -1 2014-07-01 16:55:53,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: read oid 225 snap 86 2014-07-01 16:55:53,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: write oid 440 current snap is 89 2014-07-01 16:55:53,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: seq_num 695 ranges {515551=400000} 2014-07-01 16:55:53,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: writing mira0437250-440 from 515551 to 915551 tid 1 2014-07-01 16:55:53,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: snap_create 2014-07-01 16:55:53,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: expect (ObjNum 506 snap 3 seq_num 506) 2014-07-01 16:55:54,327.327 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:55,000.000 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:55:44.723915', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881835964, u'health': u'HEALTH_OK', u'kb_used': 14945088, u'store_stats': {u'bytes_total': 5988345, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3891177}}, {u'last_updated': u'2014-07-01 07:55:54.541454', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885849936, u'health': u'HEALTH_OK', u'kb_used': 10931116, u'store_stats': {u'bytes_total': 8084542, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890222}}, {u'last_updated': u'2014-07-01 07:55:11.323787', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886021776, u'health': u'HEALTH_OK', u'kb_used': 10759276, u'store_stats': {u'bytes_total': 5108638, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4584334}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'17 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 255/2977 objects degraded (8.566%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 255, u'num_pgs': 434, u'data_bytes': 2399903139, u'degraded_total': 2977, u'bytes_used': 5388570624, u'version': 1946, u'pgs_by_state': [{u'count': 4, u'state_name': u'active+recovery_wait'}, {u'count': 413, u'state_name': u'active+clean'}, {u'count': 17, u'state_name': u'active+recovering'}], u'degraded_ratio': u'8.566', u'bytes_avail': 4993193967616}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 790, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:55:55,001.001 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:55:55,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: expect (ObjNum 649 snap 69 seq_num 649) 2014-07-01 16:55:55,785.785 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:55:55,786.786 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:55:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: finishing write tid 1 to mira0437250-440 2014-07-01 16:55:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: finishing write tid 3 to mira0437250-440 2014-07-01 16:55:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: expect (ObjNum 224 snap 0 seq_num 224) 2014-07-01 16:55:56,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: finishing write tid 4 to mira0437250-440 2014-07-01 16:55:56,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 262 (ObjNum 695 snap 89 seq_num 695) dirty exists 2014-07-01 16:55:56,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1020: done (4 left) 2014-07-01 16:55:56,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1021: done (3 left) 2014-07-01 16:55:56,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1022: done (2 left) 2014-07-01 16:55:56,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1023: done (1 left) 2014-07-01 16:55:56,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1024: done (0 left) 2014-07-01 16:55:56,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1025: snap_remove snap 88 2014-07-01 16:55:56,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1025: done (0 left) 2014-07-01 16:55:56,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: rollback oid 225 current snap is 90 2014-07-01 16:55:56,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 225 to 42 2014-07-01 16:55:56,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1027: snap_remove snap 61 2014-07-01 16:55:56,589.589 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:55:56,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 246 (ObjNum 224 snap 0 seq_num 224) dirty exists 2014-07-01 16:55:58,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: finishing write tid 1 to mira0435601-111 2014-07-01 16:55:58,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: finishing write tid 2 to mira0435601-111 2014-07-01 16:55:58,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: finishing write tid 4 to mira0435601-111 2014-07-01 16:55:58,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: finishing write tid 5 to mira0435601-111 2014-07-01 16:55:58,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 178 (ObjNum 1702 snap 0 seq_num 1702) dirty exists 2014-07-01 16:55:58,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2685: done (6 left) 2014-07-01 16:55:58,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2687: done (5 left) 2014-07-01 16:55:58,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: done (4 left) 2014-07-01 16:55:58,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2690: done (3 left) 2014-07-01 16:55:58,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 321 current snap is 0 2014-07-01 16:55:58,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: seq_num 1705 ranges {5595136=331776} 2014-07-01 16:55:58,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: writing mira0435601-321 from 5595136 to 5926912 tid 1 2014-07-01 16:55:58,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 68 current snap is 0 2014-07-01 16:55:58,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: seq_num 1706 ranges {3403776=204800} 2014-07-01 16:55:58,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: writing mira0435601-68 from 3403776 to 3608576 tid 1 2014-07-01 16:55:58,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 402 current snap is 0 2014-07-01 16:55:58,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: seq_num 1707 ranges {0=73728} 2014-07-01 16:55:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: writing mira0435601-402 from 0 to 73728 tid 1 2014-07-01 16:55:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:2696: delete oid 373 current snap is 0 2014-07-01 16:55:58,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2696: done (6 left) 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2697: read oid 366 snap -1 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2698: read oid 222 snap -1 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: read oid 239 snap -1 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 384 current snap is 0 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: seq_num 1709 ranges {3579904=425984,4005888=569344,4575232=577536,5152768=458752} 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: writing mira0435601-384 from 3579904 to 4005888 tid 1 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: writing mira0435601-384 from 4005888 to 4575232 tid 2 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: writing mira0435601-384 from 4575232 to 5152768 tid 3 2014-07-01 16:55:58,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: writing mira0435601-384 from 5152768 to 5611520 tid 4 2014-07-01 16:55:58,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: read oid 81 snap -1 2014-07-01 16:55:58,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:2702: read oid 71 snap -1 2014-07-01 16:55:58,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: delete oid 438 current snap is 0 2014-07-01 16:55:59,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: expect (ObjNum 80 snap 0 seq_num 80) 2014-07-01 16:55:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1026: done (1 left) 2014-07-01 16:55:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1027: done (0 left) 2014-07-01 16:55:59,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: write oid 93 current snap is 90 2014-07-01 16:55:59,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: seq_num 696 ranges {610780=1} 2014-07-01 16:55:59,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: writing mira0437250-93 from 610780 to 610781 tid 1 2014-07-01 16:55:59,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: read oid 127 snap 89 2014-07-01 16:55:59,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: read oid 125 snap -1 2014-07-01 16:55:59,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: read oid 321 snap -1 2014-07-01 16:55:59,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: read oid 235 snap -1 2014-07-01 16:55:59,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: write oid 349 current snap is 90 2014-07-01 16:55:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: seq_num 697 ranges {307033=1} 2014-07-01 16:55:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: writing mira0437250-349 from 307033 to 307034 tid 1 2014-07-01 16:55:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: snap_create 2014-07-01 16:55:59,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: expect (ObjNum 234 snap 0 seq_num 234) 2014-07-01 16:56:00,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: expect (ObjNum 506 snap 3 seq_num 506) 2014-07-01 16:56:01,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: finishing write tid 1 to mira0435601-388 2014-07-01 16:56:01,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: finishing write tid 3 to mira0435601-388 2014-07-01 16:56:01,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 1 to mira0435601-264 2014-07-01 16:56:01,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 3 to mira0435601-264 2014-07-01 16:56:01,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 4 to mira0435601-264 2014-07-01 16:56:01,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 313 (ObjNum 1703 snap 0 seq_num 1703) dirty exists 2014-07-01 16:56:01,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 1 to mira0435601-321 2014-07-01 16:56:01,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2692: expect (ObjNum 1415 snap 0 seq_num 1415) 2014-07-01 16:56:01,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: finishing write tid 4 to mira0435601-388 2014-07-01 16:56:01,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 248 (ObjNum 1704 snap 0 seq_num 1704) dirty exists 2014-07-01 16:56:01,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: expect (ObjNum 1620 snap 0 seq_num 1620) 2014-07-01 16:56:01,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 1 to mira0435601-384 2014-07-01 16:56:01,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 2 to mira0435601-384 2014-07-01 16:56:01,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 3 to mira0435601-384 2014-07-01 16:56:01,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2697: expect (ObjNum 1285 snap 0 seq_num 1285) 2014-07-01 16:56:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: expect (ObjNum 126 snap 0 seq_num 126) 2014-07-01 16:56:02,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: finishing write tid 1 to mira0437250-349 2014-07-01 16:56:02,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: finishing write tid 3 to mira0437250-349 2014-07-01 16:56:02,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: finishing write tid 4 to mira0437250-349 2014-07-01 16:56:02,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 236 (ObjNum 697 snap 90 seq_num 697) dirty exists 2014-07-01 16:56:02,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: expect (ObjNum 630 snap 57 seq_num 630) 2014-07-01 16:56:02,355.355 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:03,131.131 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:03,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: finishing write tid 1 to mira0435601-68 2014-07-01 16:56:03,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: finishing write tid 3 to mira0435601-68 2014-07-01 16:56:03,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: finishing write tid 1 to mira0435601-402 2014-07-01 16:56:03,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 3 to mira0435601-321 2014-07-01 16:56:03,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: finishing write tid 3 to mira0435601-402 2014-07-01 16:56:03,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 4 to mira0435601-384 2014-07-01 16:56:03,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 6 to mira0435601-384 2014-07-01 16:56:03,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: finishing write tid 4 to mira0435601-68 2014-07-01 16:56:03,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 152 (ObjNum 1706 snap 0 seq_num 1706) dirty exists 2014-07-01 16:56:03,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: finishing write tid 4 to mira0435601-402 2014-07-01 16:56:03,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 280 (ObjNum 1707 snap 0 seq_num 1707) dirty exists 2014-07-01 16:56:03,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: finishing write tid 7 to mira0435601-384 2014-07-01 16:56:03,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 271 (ObjNum 1709 snap 0 seq_num 1709) dirty exists 2014-07-01 16:56:03,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2702: expect (ObjNum 1634 snap 0 seq_num 1634) 2014-07-01 16:56:03,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: finishing write tid 1 to mira0437250-93 2014-07-01 16:56:03,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: finishing write tid 3 to mira0437250-93 2014-07-01 16:56:03,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: finishing write tid 4 to mira0437250-93 2014-07-01 16:56:03,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 180 (ObjNum 696 snap 90 seq_num 696) dirty exists 2014-07-01 16:56:03,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1028: done (6 left) 2014-07-01 16:56:03,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1029: done (5 left) 2014-07-01 16:56:03,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1030: done (4 left) 2014-07-01 16:56:03,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1031: done (3 left) 2014-07-01 16:56:03,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:1032: done (2 left) 2014-07-01 16:56:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1033: done (1 left) 2014-07-01 16:56:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1034: done (0 left) 2014-07-01 16:56:03,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1035: snap_create 2014-07-01 16:56:05,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1035: done (0 left) 2014-07-01 16:56:05,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1036: delete oid 179 current snap is 92 2014-07-01 16:56:05,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:1036: done (0 left) 2014-07-01 16:56:05,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:1037: rollback oid 447 current snap is 92 2014-07-01 16:56:05,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 447 to 50 2014-07-01 16:56:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: delete oid 451 current snap is 92 2014-07-01 16:56:05,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 447 v 0 (ObjNum 1 snap 1 seq_num 37058336) dirty dne 2014-07-01 16:56:05,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:1037: done (1 left) 2014-07-01 16:56:05,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:1038: done (0 left) 2014-07-01 16:56:05,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: write oid 435 current snap is 92 2014-07-01 16:56:05,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: seq_num 700 ranges {408276=643615,1812238=196038} 2014-07-01 16:56:05,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0437250-435 from 408276 to 1051891 tid 1 2014-07-01 16:56:05,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: writing mira0437250-435 from 1812238 to 2008276 tid 2 2014-07-01 16:56:05,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: read oid 445 snap -1 2014-07-01 16:56:05,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: write oid 312 current snap is 92 2014-07-01 16:56:05,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: seq_num 701 ranges {752081=499068,1657570=443095,2752080=1} 2014-07-01 16:56:05,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: writing mira0437250-312 from 752081 to 1251149 tid 1 2014-07-01 16:56:05,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: writing mira0437250-312 from 1657570 to 2100665 tid 2 2014-07-01 16:56:05,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: writing mira0437250-312 from 2752080 to 2752081 tid 3 2014-07-01 16:56:05,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: read oid 382 snap -1 2014-07-01 16:56:05,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: expect (ObjNum 444 snap 0 seq_num 444) 2014-07-01 16:56:05,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 4 to mira0435601-321 2014-07-01 16:56:05,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 214 (ObjNum 1705 snap 0 seq_num 1705) dirty exists 2014-07-01 16:56:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: done (12 left) 2014-07-01 16:56:05,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: done (11 left) 2014-07-01 16:56:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2692: done (10 left) 2014-07-01 16:56:05,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: done (9 left) 2014-07-01 16:56:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: done (8 left) 2014-07-01 16:56:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: done (7 left) 2014-07-01 16:56:05,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2697: done (6 left) 2014-07-01 16:56:05,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: done (5 left) 2014-07-01 16:56:05,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: done (4 left) 2014-07-01 16:56:05,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: done (3 left) 2014-07-01 16:56:05,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2702: done (2 left) 2014-07-01 16:56:05,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: done (1 left) 2014-07-01 16:56:05,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2704: read oid 25 snap -1 2014-07-01 16:56:05,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: read oid 332 snap -1 2014-07-01 16:56:05,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 314 current snap is 0 2014-07-01 16:56:05,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: seq_num 1711 ranges {0=724992} 2014-07-01 16:56:05,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: finishing write tid 1 to mira0437250-312 2014-07-01 16:56:05,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: finishing write tid 2 to mira0437250-312 2014-07-01 16:56:05,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: finishing write tid 3 to mira0437250-312 2014-07-01 16:56:05,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: finishing write tid 5 to mira0437250-312 2014-07-01 16:56:05,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: finishing write tid 6 to mira0437250-312 2014-07-01 16:56:05,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 355 (ObjNum 701 snap 92 seq_num 701) dirty exists 2014-07-01 16:56:05,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: expect (ObjNum 381 snap 0 seq_num 381) 2014-07-01 16:56:05,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: writing mira0435601-314 from 0 to 724992 tid 1 2014-07-01 16:56:05,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: read oid 283 snap -1 2014-07-01 16:56:05,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 312 current snap is 0 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: seq_num 1712 ranges {454656=598016} 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: writing mira0435601-312 from 454656 to 1052672 tid 1 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: read oid 47 snap -1 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 460 current snap is 0 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: seq_num 1713 ranges {2850816=622592,3473408=794624,4268032=409600} 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: writing mira0435601-460 from 2850816 to 3473408 tid 1 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: writing mira0435601-460 from 3473408 to 4268032 tid 2 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: writing mira0435601-460 from 4268032 to 4677632 tid 3 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2711: delete oid 303 current snap is 0 2014-07-01 16:56:05,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2704: expect (ObjNum 1331 snap 0 seq_num 1331) 2014-07-01 16:56:05,899.899 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:56:06,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 1 to mira0437250-435 2014-07-01 16:56:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 2 to mira0437250-435 2014-07-01 16:56:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 4 to mira0437250-435 2014-07-01 16:56:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: finishing write tid 5 to mira0437250-435 2014-07-01 16:56:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 246 (ObjNum 700 snap 92 seq_num 700) dirty exists 2014-07-01 16:56:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1039: done (3 left) 2014-07-01 16:56:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1040: done (2 left) 2014-07-01 16:56:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1041: done (1 left) 2014-07-01 16:56:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1042: done (0 left) 2014-07-01 16:56:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1043: rollback oid 404 current snap is 92 2014-07-01 16:56:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 404 to 91 2014-07-01 16:56:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: write oid 20 current snap is 92 2014-07-01 16:56:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: seq_num 702 ranges {696911=622296,1736630=664714,3094083=402828} 2014-07-01 16:56:06,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: writing mira0437250-20 from 696911 to 1319207 tid 1 2014-07-01 16:56:06,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: writing mira0437250-20 from 1736630 to 2401344 tid 2 2014-07-01 16:56:06,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: writing mira0437250-20 from 3094083 to 3496911 tid 3 2014-07-01 16:56:06,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: read oid 437 snap -1 2014-07-01 16:56:06,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1046: snap_remove snap 59 2014-07-01 16:56:06,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: expect (ObjNum 436 snap 0 seq_num 436) 2014-07-01 16:56:06,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: finishing write tid 1 to mira0435601-314 2014-07-01 16:56:06,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: finishing write tid 3 to mira0435601-314 2014-07-01 16:56:06,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: expect (ObjNum 1461 snap 0 seq_num 1461) 2014-07-01 16:56:06,565.565 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:55:44.723915', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881835964, u'health': u'HEALTH_OK', u'kb_used': 14945088, u'store_stats': {u'bytes_total': 5988345, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3891177}}, {u'last_updated': u'2014-07-01 07:55:54.541454', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885849936, u'health': u'HEALTH_OK', u'kb_used': 10931116, u'store_stats': {u'bytes_total': 8084542, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890222}}, {u'last_updated': u'2014-07-01 07:55:11.323787', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 886021776, u'health': u'HEALTH_OK', u'kb_used': 10759276, u'store_stats': {u'bytes_total': 5108638, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4584334}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'9 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 119/2982 objects degraded (3.991%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'recovering_bytes_per_sec': 41455496, u'degraded_objects': 119, u'num_pgs': 434, u'recovering_keys_per_sec': 0, u'data_bytes': 2394707761, u'degraded_total': 2982, u'bytes_used': 5454495744, u'recovering_objects_per_sec': 13, u'write_bytes_sec': 1323803, u'op_per_sec': 9, u'version': 1953, u'pgs_by_state': [{u'count': 425, u'state_name': u'active+clean'}, {u'count': 9, u'state_name': u'active+recovering'}], u'read_bytes_sec': 5120834, u'degraded_ratio': u'3.991', u'bytes_avail': 4993128042496}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 794, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:56:06,566.566 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:56:06,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 1 to mira0437250-20 2014-07-01 16:56:06,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 2 to mira0437250-20 2014-07-01 16:56:06,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 3 to mira0437250-20 2014-07-01 16:56:06,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 5 to mira0437250-20 2014-07-01 16:56:06,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:1045: done (3 left) 2014-07-01 16:56:06,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:1046: done (2 left) 2014-07-01 16:56:06,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:1047: snap_create 2014-07-01 16:56:06,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: finishing write tid 6 to mira0437250-20 2014-07-01 16:56:06,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 351 (ObjNum 702 snap 92 seq_num 702) dirty exists 2014-07-01 16:56:06,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: finishing write tid 4 to mira0435601-314 2014-07-01 16:56:06,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 203 (ObjNum 1711 snap 0 seq_num 1711) dirty exists 2014-07-01 16:56:06,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: expect (ObjNum 46 snap 0 seq_num 46) 2014-07-01 16:56:06,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 1 to mira0435601-312 2014-07-01 16:56:06,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 3 to mira0435601-312 2014-07-01 16:56:06,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: finishing write tid 1 to mira0435601-460 2014-07-01 16:56:06,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: expect (ObjNum 1301 snap 0 seq_num 1301) 2014-07-01 16:56:07,412.412 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:56:07,412.412 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:07,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1044: done (2 left) 2014-07-01 16:56:07,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1047: done (1 left) 2014-07-01 16:56:07,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: delete oid 235 current snap is 93 2014-07-01 16:56:08,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 267 (ObjNum 537 snap 14 seq_num 537) dirty exists 2014-07-01 16:56:09,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 4 to mira0435601-312 2014-07-01 16:56:09,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 149 (ObjNum 1712 snap 0 seq_num 1712) dirty exists 2014-07-01 16:56:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:2698: expect (ObjNum 1455 snap 0 seq_num 1455) 2014-07-01 16:56:09,190.190 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:12,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: finishing write tid 2 to mira0435601-460 2014-07-01 16:56:12,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: finishing write tid 3 to mira0435601-460 2014-07-01 16:56:12,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: finishing write tid 5 to mira0435601-460 2014-07-01 16:56:12,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: finishing write tid 6 to mira0435601-460 2014-07-01 16:56:12,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 153 (ObjNum 1713 snap 0 seq_num 1713) dirty exists 2014-07-01 16:56:12,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2698: done (8 left) 2014-07-01 16:56:12,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2704: done (7 left) 2014-07-01 16:56:12,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: done (6 left) 2014-07-01 16:56:12,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: done (5 left) 2014-07-01 16:56:12,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: done (4 left) 2014-07-01 16:56:12,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: done (3 left) 2014-07-01 16:56:12,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: done (2 left) 2014-07-01 16:56:12,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: done (1 left) 2014-07-01 16:56:12,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2711: done (0 left) 2014-07-01 16:56:12,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 484 current snap is 0 2014-07-01 16:56:12,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: seq_num 1715 ranges {3342336=16384} 2014-07-01 16:56:12,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: writing mira0435601-484 from 3342336 to 3358720 tid 1 2014-07-01 16:56:12,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 16:56:12,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: seq_num 1716 ranges {6103040=442368,6545408=593920,7139328=208896} 2014-07-01 16:56:12,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: writing mira0435601-97 from 6103040 to 6545408 tid 1 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: writing mira0435601-97 from 6545408 to 7139328 tid 2 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: writing mira0435601-97 from 7139328 to 7348224 tid 3 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 19 current snap is 0 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: seq_num 1717 ranges {1593344=716800,2310144=401408} 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: writing mira0435601-19 from 1593344 to 2310144 tid 1 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: writing mira0435601-19 from 2310144 to 2711552 tid 2 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2715: delete oid 92 current snap is 0 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2715: done (3 left) 2014-07-01 16:56:12,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: read oid 480 snap -1 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2717: read oid 327 snap -1 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 413 current snap is 0 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: seq_num 1719 ranges {3485696=663552} 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1043: done (1 left) 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1048: done (0 left) 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: write oid 448 current snap is 93 2014-07-01 16:56:12,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: seq_num 704 ranges {452298=1} 2014-07-01 16:56:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: writing mira0437250-448 from 452298 to 452299 tid 1 2014-07-01 16:56:12,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: snap_remove snap 84 2014-07-01 16:56:12,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 1 to mira0437250-448 2014-07-01 16:56:12,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: writing mira0435601-413 from 3485696 to 4149248 tid 1 2014-07-01 16:56:12,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2719: read oid 399 snap -1 2014-07-01 16:56:12,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: expect (ObjNum 1588 snap 0 seq_num 1588) 2014-07-01 16:56:12,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 3 to mira0437250-448 2014-07-01 16:56:12,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: finishing write tid 4 to mira0437250-448 2014-07-01 16:56:12,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 289 (ObjNum 704 snap 93 seq_num 704) dirty exists 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1049: done (1 left) 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1050: done (0 left) 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1051: rollback oid 228 current snap is 93 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 228 to 69 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1052: snap_remove snap 83 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 361 (ObjNum 227 snap 0 seq_num 227) dirty exists 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: finishing write tid 1 to mira0435601-97 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: finishing write tid 1 to mira0435601-484 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: finishing write tid 2 to mira0435601-97 2014-07-01 16:56:12,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: finishing write tid 3 to mira0435601-97 2014-07-01 16:56:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: finishing write tid 5 to mira0435601-97 2014-07-01 16:56:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: finishing write tid 6 to mira0435601-97 2014-07-01 16:56:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 284 (ObjNum 1716 snap 0 seq_num 1716) dirty exists 2014-07-01 16:56:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: done (6 left) 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: done (5 left) 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2719: done (4 left) 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: read oid 479 snap -1 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: read oid 461 snap -1 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: seq_num 1720 ranges {5242880=536576} 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: writing mira0435601-292 from 5242880 to 5779456 tid 1 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2723: read oid 34 snap -1 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: finishing write tid 3 to mira0435601-484 2014-07-01 16:56:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: expect (ObjNum 1562 snap 0 seq_num 1562) 2014-07-01 16:56:13,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: finishing write tid 1 to mira0435601-19 2014-07-01 16:56:13,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: finishing write tid 2 to mira0435601-19 2014-07-01 16:56:13,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: expect (ObjNum 1225 snap 0 seq_num 1225) 2014-07-01 16:56:15,099.099 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:15,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: finishing write tid 4 to mira0435601-19 2014-07-01 16:56:15,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2717: expect (ObjNum 1695 snap 0 seq_num 1695) 2014-07-01 16:56:17,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: finishing write tid 4 to mira0435601-484 2014-07-01 16:56:17,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 315 (ObjNum 1715 snap 0 seq_num 1715) dirty exists 2014-07-01 16:56:17,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: finishing write tid 1 to mira0435601-413 2014-07-01 16:56:17,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: finishing write tid 3 to mira0435601-413 2014-07-01 16:56:17,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: done (7 left) 2014-07-01 16:56:17,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2717: done (6 left) 2014-07-01 16:56:17,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: done (5 left) 2014-07-01 16:56:17,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: done (4 left) 2014-07-01 16:56:17,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 83 current snap is 0 2014-07-01 16:56:17,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: seq_num 1721 ranges {2060288=409600} 2014-07-01 16:56:17,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: writing mira0435601-83 from 2060288 to 2469888 tid 1 2014-07-01 16:56:17,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 133 current snap is 0 2014-07-01 16:56:17,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: seq_num 1722 ranges {2142208=434176,2576384=741376,3317760=462848} 2014-07-01 16:56:17,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: writing mira0435601-133 from 2142208 to 2576384 tid 1 2014-07-01 16:56:17,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: writing mira0435601-133 from 2576384 to 3317760 tid 2 2014-07-01 16:56:17,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: writing mira0435601-133 from 3317760 to 3780608 tid 3 2014-07-01 16:56:17,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 237 current snap is 0 2014-07-01 16:56:17,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: seq_num 1723 ranges {0=757760,757760=569344,1327104=659456,1986560=376832} 2014-07-01 16:56:17,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: writing mira0435601-237 from 0 to 757760 tid 1 2014-07-01 16:56:17,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: writing mira0435601-237 from 757760 to 1327104 tid 2 2014-07-01 16:56:17,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: writing mira0435601-237 from 1327104 to 1986560 tid 3 2014-07-01 16:56:17,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: writing mira0435601-237 from 1986560 to 2363392 tid 4 2014-07-01 16:56:17,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2727: read oid 174 snap -1 2014-07-01 16:56:17,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: finishing write tid 4 to mira0435601-413 2014-07-01 16:56:17,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 278 (ObjNum 1719 snap 0 seq_num 1719) dirty exists 2014-07-01 16:56:17,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: finishing write tid 1 to mira0435601-292 2014-07-01 16:56:17,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: finishing write tid 3 to mira0435601-292 2014-07-01 16:56:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: done (7 left) 2014-07-01 16:56:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: read oid 23 snap -1 2014-07-01 16:56:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: finishing write tid 5 to mira0435601-19 2014-07-01 16:56:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 156 (ObjNum 1717 snap 0 seq_num 1717) dirty exists 2014-07-01 16:56:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2723: expect (ObjNum 1586 snap 0 seq_num 1586) 2014-07-01 16:56:19,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:1051: done (1 left) 2014-07-01 16:56:19,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:1052: done (0 left) 2014-07-01 16:56:19,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: delete oid 210 current snap is 93 2014-07-01 16:56:19,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1053: done (0 left) 2014-07-01 16:56:19,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: write oid 332 current snap is 93 2014-07-01 16:56:19,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: seq_num 706 ranges {492358=400000} 2014-07-01 16:56:19,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: writing mira0437250-332 from 492358 to 892358 tid 1 2014-07-01 16:56:19,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:1055: read oid 121 snap -1 2014-07-01 16:56:19,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: write oid 437 current snap is 93 2014-07-01 16:56:19,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: seq_num 707 ranges {437188=670886,1637187=1} 2014-07-01 16:56:19,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0437250-437 from 437188 to 1108074 tid 1 2014-07-01 16:56:19,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: writing mira0437250-437 from 1637187 to 1637188 tid 2 2014-07-01 16:56:19,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: snap_remove snap 70 2014-07-01 16:56:19,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: finishing write tid 1 to mira0437250-332 2014-07-01 16:56:19,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1055: expect (ObjNum 120 snap 0 seq_num 120) 2014-07-01 16:56:19,639.639 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:19,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: finishing write tid 3 to mira0437250-332 2014-07-01 16:56:19,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: finishing write tid 4 to mira0437250-332 2014-07-01 16:56:19,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 287 (ObjNum 706 snap 93 seq_num 706) dirty exists 2014-07-01 16:56:19,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 1 to mira0437250-437 2014-07-01 16:56:19,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 2 to mira0437250-437 2014-07-01 16:56:19,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 4 to mira0437250-437 2014-07-01 16:56:19,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: finishing write tid 5 to mira0437250-437 2014-07-01 16:56:19,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 291 (ObjNum 707 snap 93 seq_num 707) dirty exists 2014-07-01 16:56:20,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: finishing write tid 4 to mira0435601-292 2014-07-01 16:56:20,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 273 (ObjNum 1720 snap 0 seq_num 1720) dirty exists 2014-07-01 16:56:20,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2727: expect (ObjNum 1655 snap 0 seq_num 1655) 2014-07-01 16:56:20,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1054: done (3 left) 2014-07-01 16:56:20,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1055: done (2 left) 2014-07-01 16:56:20,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1056: done (1 left) 2014-07-01 16:56:20,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1057: done (0 left) 2014-07-01 16:56:20,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1058: snap_remove snap 69 2014-07-01 16:56:21,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1058: done (0 left) 2014-07-01 16:56:21,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1059: snap_remove snap 50 2014-07-01 16:56:22,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1059: done (0 left) 2014-07-01 16:56:22,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1060: snap_remove snap 52 2014-07-01 16:56:22,471.471 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:56:23,212.212 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:55:44.723915', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881835964, u'health': u'HEALTH_OK', u'kb_used': 14945088, u'store_stats': {u'bytes_total': 5988345, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3891177}}, {u'last_updated': u'2014-07-01 07:55:54.541454', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885849936, u'health': u'HEALTH_OK', u'kb_used': 10931116, u'store_stats': {u'bytes_total': 8084542, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890222}}, {u'last_updated': u'2014-07-01 07:56:11.324131', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885826388, u'health': u'HEALTH_OK', u'kb_used': 10954664, u'store_stats': {u'bytes_total': 10181755, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890283}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 8/2991 objects degraded (0.267%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'degraded_objects': 8, u'num_pgs': 434, u'data_bytes': 2403581128, u'degraded_total': 2991, u'bytes_used': 5285498880, u'version': 1963, u'pgs_by_state': [{u'count': 433, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'active+recovering'}], u'degraded_ratio': u'0.267', u'bytes_avail': 4993297039360}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 801, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:56:23,213.213 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:56:23,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1060: done (0 left) 2014-07-01 16:56:23,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: snap_create 2014-07-01 16:56:23,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 1 to mira0435601-83 2014-07-01 16:56:23,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 3 to mira0435601-83 2014-07-01 16:56:23,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: finishing write tid 1 to mira0435601-133 2014-07-01 16:56:23,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: finishing write tid 2 to mira0435601-133 2014-07-01 16:56:23,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: finishing write tid 3 to mira0435601-133 2014-07-01 16:56:23,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 1 to mira0435601-237 2014-07-01 16:56:23,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: done (7 left) 2014-07-01 16:56:23,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: done (6 left) 2014-07-01 16:56:23,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2723: done (5 left) 2014-07-01 16:56:23,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2727: done (4 left) 2014-07-01 16:56:23,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2729: read oid 40 snap -1 2014-07-01 16:56:23,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 2 to mira0435601-237 2014-07-01 16:56:23,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: finishing write tid 5 to mira0435601-133 2014-07-01 16:56:23,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 3 to mira0435601-237 2014-07-01 16:56:23,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 4 to mira0435601-237 2014-07-01 16:56:23,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 6 to mira0435601-237 2014-07-01 16:56:23,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 4 to mira0435601-83 2014-07-01 16:56:23,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 250 (ObjNum 1721 snap 0 seq_num 1721) dirty exists 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: done (4 left) 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2730: read oid 139 snap -1 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: finishing write tid 7 to mira0435601-237 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 118 (ObjNum 1723 snap 0 seq_num 1723) dirty exists 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: done (4 left) 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2731: read oid 256 snap -1 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: finishing write tid 6 to mira0435601-133 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 210 (ObjNum 1722 snap 0 seq_num 1722) dirty exists 2014-07-01 16:56:23,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: done (4 left) 2014-07-01 16:56:23,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 379 current snap is 0 2014-07-01 16:56:23,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: seq_num 1724 ranges {1863680=626688,2490368=802816,3293184=536576,3829760=266240} 2014-07-01 16:56:23,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: writing mira0435601-379 from 1863680 to 2490368 tid 1 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: writing mira0435601-379 from 2490368 to 3293184 tid 2 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: writing mira0435601-379 from 3293184 to 3829760 tid 3 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: writing mira0435601-379 from 3829760 to 4096000 tid 4 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 489 current snap is 0 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: seq_num 1725 ranges {4804608=655360,5459968=401408} 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: writing mira0435601-489 from 4804608 to 5459968 tid 1 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: writing mira0435601-489 from 5459968 to 5861376 tid 2 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: read oid 11 snap -1 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 268 current snap is 0 2014-07-01 16:56:23,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: seq_num 1726 ranges {2609152=684032,3293184=565248,3858432=483328,4341760=557056} 2014-07-01 16:56:23,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: writing mira0435601-268 from 2609152 to 3293184 tid 1 2014-07-01 16:56:23,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: writing mira0435601-268 from 3293184 to 3858432 tid 2 2014-07-01 16:56:23,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: writing mira0435601-268 from 3858432 to 4341760 tid 3 2014-07-01 16:56:23,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: writing mira0435601-268 from 4341760 to 4898816 tid 4 2014-07-01 16:56:23,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2736: read oid 15 snap -1 2014-07-01 16:56:23,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: expect (ObjNum 1382 snap 0 seq_num 1382) 2014-07-01 16:56:23,995.995 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:56:23,995.995 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:24,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:1061: done (0 left) 2014-07-01 16:56:24,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1062: read oid 251 snap -1 2014-07-01 16:56:24,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1063: read oid 120 snap -1 2014-07-01 16:56:24,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1064: rollback oid 389 current snap is 94 2014-07-01 16:56:24,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 389 to 34 2014-07-01 16:56:24,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: read oid 195 snap 35 2014-07-01 16:56:24,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: write oid 453 current snap is 94 2014-07-01 16:56:24,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: seq_num 708 ranges {498370=642690,1915184=472382,2800637=97733} 2014-07-01 16:56:24,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: writing mira0437250-453 from 498370 to 1141060 tid 1 2014-07-01 16:56:24,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: writing mira0437250-453 from 1915184 to 2387566 tid 2 2014-07-01 16:56:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: writing mira0437250-453 from 2800637 to 2898370 tid 3 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: delete oid 155 current snap is 94 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: finishing write tid 1 to mira0437250-453 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1067: done (5 left) 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: snap_create 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: finishing write tid 2 to mira0437250-453 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: finishing write tid 3 to mira0437250-453 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: finishing write tid 5 to mira0437250-453 2014-07-01 16:56:24,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1062: expect (ObjNum 250 snap 0 seq_num 250) 2014-07-01 16:56:24,805.805 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:24,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 327 (ObjNum 388 snap 0 seq_num 388) dirty exists 2014-07-01 16:56:24,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: finishing write tid 6 to mira0437250-453 2014-07-01 16:56:24,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 328 (ObjNum 708 snap 94 seq_num 708) dirty exists 2014-07-01 16:56:24,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:1063: expect (ObjNum 560 snap 26 seq_num 560) 2014-07-01 16:56:25,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: expect (ObjNum 194 snap 0 seq_num 194) 2014-07-01 16:56:25,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2729: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 1 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 2 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 3 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 4 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 6 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: finishing write tid 7 to mira0435601-379 2014-07-01 16:56:25,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 245 (ObjNum 1724 snap 0 seq_num 1724) dirty exists 2014-07-01 16:56:25,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: finishing write tid 1 to mira0435601-489 2014-07-01 16:56:25,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: finishing write tid 2 to mira0435601-489 2014-07-01 16:56:25,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: finishing write tid 4 to mira0435601-489 2014-07-01 16:56:25,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: finishing write tid 5 to mira0435601-489 2014-07-01 16:56:25,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 287 (ObjNum 1725 snap 0 seq_num 1725) dirty exists 2014-07-01 16:56:25,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: expect (ObjNum 1623 snap 0 seq_num 1623) 2014-07-01 16:56:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1062: done (5 left) 2014-07-01 16:56:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1063: done (4 left) 2014-07-01 16:56:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1064: done (3 left) 2014-07-01 16:56:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1065: done (2 left) 2014-07-01 16:56:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1066: done (1 left) 2014-07-01 16:56:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1068: done (0 left) 2014-07-01 16:56:26,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: read oid 119 snap -1 2014-07-01 16:56:26,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: read oid 145 snap -1 2014-07-01 16:56:26,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: snap_remove snap 82 2014-07-01 16:56:26,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: expect (ObjNum 118 snap 0 seq_num 118) 2014-07-01 16:56:27,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: expect (ObjNum 144 snap 0 seq_num 144) 2014-07-01 16:56:28,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:1069: done (2 left) 2014-07-01 16:56:28,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:1070: done (1 left) 2014-07-01 16:56:28,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:1071: done (0 left) 2014-07-01 16:56:28,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: read oid 28 snap -1 2014-07-01 16:56:28,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: read oid 416 snap -1 2014-07-01 16:56:28,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: rollback oid 138 current snap is 95 2014-07-01 16:56:28,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 138 to 91 2014-07-01 16:56:28,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:1075: rollback oid 308 current snap is 95 2014-07-01 16:56:28,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 308 to 58 2014-07-01 16:56:28,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1076: rollback oid 285 current snap is 95 2014-07-01 16:56:28,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 285 to 55 2014-07-01 16:56:28,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: snap_remove snap 42 2014-07-01 16:56:28,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 0 (ObjNum 36329728 snap 0 seq_num 36330016) dirty dne 2014-07-01 16:56:28,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 138 v 306 (ObjNum 137 snap 0 seq_num 137) dirty exists 2014-07-01 16:56:28,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: expect (ObjNum 27 snap 0 seq_num 27) 2014-07-01 16:56:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 333 (ObjNum 284 snap 0 seq_num 284) dirty exists 2014-07-01 16:56:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: expect (ObjNum 527 snap 12 seq_num 527) 2014-07-01 16:56:29,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1072: done (5 left) 2014-07-01 16:56:29,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1073: done (4 left) 2014-07-01 16:56:29,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1074: done (3 left) 2014-07-01 16:56:29,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1075: done (2 left) 2014-07-01 16:56:29,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1076: done (1 left) 2014-07-01 16:56:29,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1077: done (0 left) 2014-07-01 16:56:29,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1078: delete oid 247 current snap is 95 2014-07-01 16:56:29,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1078: done (0 left) 2014-07-01 16:56:29,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: snap_remove snap 68 2014-07-01 16:56:29,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1079: done (0 left) 2014-07-01 16:56:29,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: read oid 249 snap -1 2014-07-01 16:56:29,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: rollback oid 497 current snap is 95 2014-07-01 16:56:29,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 497 to 35 2014-07-01 16:56:29,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: write oid 453 current snap is 95 2014-07-01 16:56:29,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: seq_num 711 ranges {81210=1} 2014-07-01 16:56:29,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: writing mira0437250-453 from 81210 to 81211 tid 1 2014-07-01 16:56:29,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1083: snap_remove snap 78 2014-07-01 16:56:29,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: finishing write tid 1 to mira0437250-453 2014-07-01 16:56:29,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: finishing write tid 3 to mira0437250-453 2014-07-01 16:56:29,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: finishing write tid 4 to mira0437250-453 2014-07-01 16:56:29,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 346 (ObjNum 711 snap 95 seq_num 711) dirty exists 2014-07-01 16:56:29,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: expect (ObjNum 248 snap 0 seq_num 248) 2014-07-01 16:56:29,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 401 (ObjNum 568 snap 27 seq_num 568) dirty exists 2014-07-01 16:56:30,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 1 to mira0435601-268 2014-07-01 16:56:30,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 2 to mira0435601-268 2014-07-01 16:56:30,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 3 to mira0435601-268 2014-07-01 16:56:30,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 4 to mira0435601-268 2014-07-01 16:56:30,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 6 to mira0435601-268 2014-07-01 16:56:30,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2736: expect (ObjNum 1182 snap 0 seq_num 1182) 2014-07-01 16:56:30,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1080: done (3 left) 2014-07-01 16:56:30,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1081: done (2 left) 2014-07-01 16:56:30,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1082: done (1 left) 2014-07-01 16:56:30,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1083: done (0 left) 2014-07-01 16:56:30,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: snap_remove snap 92 2014-07-01 16:56:30,537.537 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:31,310.310 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1084: done (0 left) 2014-07-01 16:56:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1085: snap_remove snap 86 2014-07-01 16:56:32,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1085: done (0 left) 2014-07-01 16:56:32,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1086: rollback oid 197 current snap is 95 2014-07-01 16:56:32,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 197 to 87 2014-07-01 16:56:32,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: write oid 16 current snap is 95 2014-07-01 16:56:32,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: seq_num 712 ranges {577464=717427,1783167=787210,3184288=593176} 2014-07-01 16:56:32,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: writing mira0437250-16 from 577464 to 1294891 tid 1 2014-07-01 16:56:32,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: writing mira0437250-16 from 1783167 to 2570377 tid 2 2014-07-01 16:56:32,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: writing mira0437250-16 from 3184288 to 3777464 tid 3 2014-07-01 16:56:32,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: delete oid 305 current snap is 95 2014-07-01 16:56:32,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 197 v 206 (ObjNum 196 snap 0 seq_num 196) dirty exists 2014-07-01 16:56:32,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: finishing write tid 1 to mira0437250-16 2014-07-01 16:56:32,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: finishing write tid 2 to mira0437250-16 2014-07-01 16:56:32,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1086: done (2 left) 2014-07-01 16:56:32,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1088: done (1 left) 2014-07-01 16:56:32,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1089: snap_create 2014-07-01 16:56:32,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: finishing write tid 3 to mira0437250-16 2014-07-01 16:56:32,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: finishing write tid 5 to mira0437250-16 2014-07-01 16:56:32,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: finishing write tid 6 to mira0437250-16 2014-07-01 16:56:32,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 211 (ObjNum 712 snap 95 seq_num 712) dirty exists 2014-07-01 16:56:32,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: finishing write tid 7 to mira0435601-268 2014-07-01 16:56:32,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 226 (ObjNum 1726 snap 0 seq_num 1726) dirty exists 2014-07-01 16:56:32,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: done (8 left) 2014-07-01 16:56:32,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2729: done (7 left) 2014-07-01 16:56:32,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2730: done (6 left) 2014-07-01 16:56:32,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2731: done (5 left) 2014-07-01 16:56:32,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: done (4 left) 2014-07-01 16:56:32,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: done (3 left) 2014-07-01 16:56:32,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: done (2 left) 2014-07-01 16:56:32,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: done (1 left) 2014-07-01 16:56:32,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:2736: done (0 left) 2014-07-01 16:56:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2737: read oid 25 snap -1 2014-07-01 16:56:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 113 current snap is 0 2014-07-01 16:56:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: seq_num 1727 ranges {2883584=446464,3330048=798720,4128768=405504} 2014-07-01 16:56:32,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: writing mira0435601-113 from 2883584 to 3330048 tid 1 2014-07-01 16:56:33,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: writing mira0435601-113 from 3330048 to 4128768 tid 2 2014-07-01 16:56:33,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: writing mira0435601-113 from 4128768 to 4534272 tid 3 2014-07-01 16:56:33,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2739: read oid 190 snap -1 2014-07-01 16:56:33,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:2737: expect (ObjNum 1331 snap 0 seq_num 1331) 2014-07-01 16:56:33,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:1087: done (1 left) 2014-07-01 16:56:33,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1089: done (0 left) 2014-07-01 16:56:33,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: write oid 152 current snap is 96 2014-07-01 16:56:33,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: seq_num 714 ranges {733875=1} 2014-07-01 16:56:33,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: writing mira0437250-152 from 733875 to 733876 tid 1 2014-07-01 16:56:33,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: read oid 452 snap 73 2014-07-01 16:56:33,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: write oid 79 current snap is 96 2014-07-01 16:56:33,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: seq_num 715 ranges {678706=668936,1478705=1} 2014-07-01 16:56:33,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0437250-79 from 678706 to 1347642 tid 1 2014-07-01 16:56:33,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: writing mira0437250-79 from 1478705 to 1478706 tid 2 2014-07-01 16:56:33,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: write oid 466 current snap is 96 2014-07-01 16:56:33,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: seq_num 716 ranges {622511=792165,2051152=171359} 2014-07-01 16:56:33,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: writing mira0437250-466 from 622511 to 1414676 tid 1 2014-07-01 16:56:33,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: writing mira0437250-466 from 2051152 to 2222511 tid 2 2014-07-01 16:56:33,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:1094: snap_create 2014-07-01 16:56:33,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: expect (ObjNum 550 snap 23 seq_num 550) 2014-07-01 16:56:33,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: finishing write tid 1 to mira0435601-113 2014-07-01 16:56:33,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: finishing write tid 2 to mira0435601-113 2014-07-01 16:56:33,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2737: done (2 left) 2014-07-01 16:56:33,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 333 current snap is 0 2014-07-01 16:56:33,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: seq_num 1728 ranges {1323008=475136} 2014-07-01 16:56:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: writing mira0435601-333 from 1323008 to 1798144 tid 1 2014-07-01 16:56:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 146 current snap is 0 2014-07-01 16:56:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: seq_num 1729 ranges {1757184=503808,2260992=610304,2871296=593920} 2014-07-01 16:56:33,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: finishing write tid 1 to mira0437250-152 2014-07-01 16:56:33,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: finishing write tid 3 to mira0437250-152 2014-07-01 16:56:33,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: finishing write tid 4 to mira0437250-152 2014-07-01 16:56:33,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 418 (ObjNum 714 snap 96 seq_num 714) dirty exists 2014-07-01 16:56:33,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 1 to mira0437250-79 2014-07-01 16:56:33,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 2 to mira0437250-79 2014-07-01 16:56:33,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 4 to mira0437250-79 2014-07-01 16:56:33,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: finishing write tid 5 to mira0437250-79 2014-07-01 16:56:33,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 350 (ObjNum 715 snap 96 seq_num 715) dirty exists 2014-07-01 16:56:33,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: finishing write tid 1 to mira0437250-466 2014-07-01 16:56:33,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: finishing write tid 2 to mira0437250-466 2014-07-01 16:56:33,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: finishing write tid 4 to mira0437250-466 2014-07-01 16:56:33,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: writing mira0435601-146 from 1757184 to 2260992 tid 1 2014-07-01 16:56:33,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: writing mira0435601-146 from 2260992 to 2871296 tid 2 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: writing mira0435601-146 from 2871296 to 3465216 tid 3 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2742: read oid 469 snap -1 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: finishing write tid 3 to mira0435601-113 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: finishing write tid 5 to mira0435601-113 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 16:56:33,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: seq_num 1730 ranges {6459392=217088} 2014-07-01 16:56:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: writing mira0435601-11 from 6459392 to 6676480 tid 1 2014-07-01 16:56:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 16:56:33,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: seq_num 1731 ranges {4927488=647168,5574656=778240,6352896=24576} 2014-07-01 16:56:33,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: writing mira0435601-351 from 4927488 to 5574656 tid 1 2014-07-01 16:56:33,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: writing mira0435601-351 from 5574656 to 6352896 tid 2 2014-07-01 16:56:33,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: writing mira0435601-351 from 6352896 to 6377472 tid 3 2014-07-01 16:56:33,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 300 current snap is 0 2014-07-01 16:56:33,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: seq_num 1732 ranges {2039808=569344,2609152=761856,3371008=425984,3796992=417792} 2014-07-01 16:56:33,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: writing mira0435601-300 from 2039808 to 2609152 tid 1 2014-07-01 16:56:34,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: writing mira0435601-300 from 2609152 to 3371008 tid 2 2014-07-01 16:56:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: writing mira0435601-300 from 3371008 to 3796992 tid 3 2014-07-01 16:56:34,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: writing mira0435601-300 from 3796992 to 4214784 tid 4 2014-07-01 16:56:34,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2746: read oid 165 snap -1 2014-07-01 16:56:34,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: finishing write tid 6 to mira0435601-113 2014-07-01 16:56:34,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 113 v 277 (ObjNum 1727 snap 0 seq_num 1727) dirty exists 2014-07-01 16:56:34,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:2739: expect (ObjNum 507 snap 0 seq_num 507) 2014-07-01 16:56:34,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: finishing write tid 5 to mira0437250-466 2014-07-01 16:56:34,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 209 (ObjNum 716 snap 96 seq_num 716) dirty exists 2014-07-01 16:56:34,240.240 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:56:34,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:1090: done (4 left) 2014-07-01 16:56:34,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:1091: done (3 left) 2014-07-01 16:56:34,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1092: done (2 left) 2014-07-01 16:56:34,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1093: done (1 left) 2014-07-01 16:56:34,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1094: done (0 left) 2014-07-01 16:56:34,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: read oid 123 snap -1 2014-07-01 16:56:34,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: read oid 281 snap -1 2014-07-01 16:56:34,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:1097: rollback oid 106 current snap is 97 2014-07-01 16:56:34,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 106 to 85 2014-07-01 16:56:34,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: delete oid 497 current snap is 97 2014-07-01 16:56:34,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: expect (ObjNum 280 snap 0 seq_num 280) 2014-07-01 16:56:34,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: expect (ObjNum 122 snap 0 seq_num 122) 2014-07-01 16:56:34,936.936 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:55:44.723915', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881835964, u'health': u'HEALTH_OK', u'kb_used': 14945088, u'store_stats': {u'bytes_total': 5988345, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3891177}}, {u'last_updated': u'2014-07-01 07:55:54.541454', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885849936, u'health': u'HEALTH_OK', u'kb_used': 10931116, u'store_stats': {u'bytes_total': 8084542, u'bytes_log': 4128768, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890222}}, {u'last_updated': u'2014-07-01 07:56:11.324131', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885826388, u'health': u'HEALTH_OK', u'kb_used': 10954664, u'store_stats': {u'bytes_total': 10181755, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 3890283}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 76 > pgp_num 16'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 4998582538240, u'num_pgs': 434, u'data_bytes': 2420290907, u'bytes_used': 5214232576, u'op_per_sec': 14, u'version': 1979, u'pgs_by_state': [{u'count': 434, u'state_name': u'active+clean'}], u'read_bytes_sec': 1998, u'write_bytes_sec': 2990327, u'bytes_avail': 4993368305664}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 812, u'num_in_osds': 5}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:56:34,937.937 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 16:56:34,937.937 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:56:34,937.937 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 16:56:34,937.937 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:56:34,937.937 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:56:34,937.937 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 16:56:35,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2742: expect (ObjNum 1508 snap 0 seq_num 1508) 2014-07-01 16:56:36,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 352 (ObjNum 105 snap 0 seq_num 105) dirty exists 2014-07-01 16:56:36,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1095: done (3 left) 2014-07-01 16:56:36,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1096: done (2 left) 2014-07-01 16:56:36,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1097: done (1 left) 2014-07-01 16:56:36,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:1098: done (0 left) 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: read oid 432 snap -1 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: rollback oid 258 current snap is 97 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 258 to 80 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: write oid 267 current snap is 97 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: seq_num 718 ranges {228522=1} 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: writing mira0437250-267 from 228522 to 228523 tid 1 2014-07-01 16:56:36,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: read oid 460 snap -1 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1103: rollback oid 334 current snap is 97 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 334 to 67 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: write oid 113 current snap is 97 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: seq_num 719 ranges {722729=743674,1922381=532849,3090020=615821,3922728=1} 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: writing mira0437250-113 from 722729 to 1466403 tid 1 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: writing mira0437250-113 from 1922381 to 2455230 tid 2 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: writing mira0437250-113 from 3090020 to 3705841 tid 3 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: writing mira0437250-113 from 3922728 to 3922729 tid 4 2014-07-01 16:56:36,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:1105: snap_remove snap 89 2014-07-01 16:56:36,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: finishing write tid 1 to mira0437250-267 2014-07-01 16:56:36,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: finishing write tid 3 to mira0437250-267 2014-07-01 16:56:36,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 278 (ObjNum 333 snap 0 seq_num 333) dirty exists 2014-07-01 16:56:36,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: expect (ObjNum 431 snap 0 seq_num 431) 2014-07-01 16:56:37,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 1 to mira0437250-113 2014-07-01 16:56:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 2 to mira0437250-113 2014-07-01 16:56:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1099: done (6 left) 2014-07-01 16:56:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1103: done (5 left) 2014-07-01 16:56:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1105: done (4 left) 2014-07-01 16:56:37,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1106: delete oid 139 current snap is 97 2014-07-01 16:56:37,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: expect (ObjNum 459 snap 0 seq_num 459) 2014-07-01 16:56:38,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 1 to mira0435601-333 2014-07-01 16:56:38,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: finishing write tid 1 to mira0435601-11 2014-07-01 16:56:38,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: done (8 left) 2014-07-01 16:56:38,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2739: done (7 left) 2014-07-01 16:56:38,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2742: done (6 left) 2014-07-01 16:56:38,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2747: read oid 102 snap -1 2014-07-01 16:56:38,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: finishing write tid 3 to mira0435601-11 2014-07-01 16:56:38,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 3 to mira0435601-333 2014-07-01 16:56:38,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 1 to mira0435601-146 2014-07-01 16:56:38,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 2 to mira0435601-146 2014-07-01 16:56:38,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 3 to mira0435601-146 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 5 to mira0435601-146 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: finishing write tid 4 to mira0435601-11 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 158 (ObjNum 1730 snap 0 seq_num 1730) dirty exists 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: finishing write tid 1 to mira0435601-351 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: done (6 left) 2014-07-01 16:56:38,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2748: read oid 70 snap -1 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: finishing write tid 2 to mira0435601-351 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: finishing write tid 3 to mira0435601-351 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: finishing write tid 5 to mira0435601-351 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 1 to mira0435601-300 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 4 to mira0435601-333 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 247 (ObjNum 1728 snap 0 seq_num 1728) dirty exists 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 2 to mira0435601-300 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 3 to mira0435601-300 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 4 to mira0435601-300 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 6 to mira0435601-300 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 6 to mira0435601-146 2014-07-01 16:56:38,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 182 (ObjNum 1729 snap 0 seq_num 1729) dirty exists 2014-07-01 16:56:38,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: finishing write tid 6 to mira0435601-351 2014-07-01 16:56:38,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 347 (ObjNum 1731 snap 0 seq_num 1731) dirty exists 2014-07-01 16:56:38,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: finishing write tid 7 to mira0435601-300 2014-07-01 16:56:38,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 195 (ObjNum 1732 snap 0 seq_num 1732) dirty exists 2014-07-01 16:56:38,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2746: expect (ObjNum 1358 snap 0 seq_num 1358) 2014-07-01 16:56:39,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 258 v 326 (ObjNum 257 snap 0 seq_num 257) dirty exists 2014-07-01 16:56:39,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 3 to mira0437250-113 2014-07-01 16:56:39,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 4 to mira0437250-113 2014-07-01 16:56:39,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 6 to mira0437250-113 2014-07-01 16:56:39,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: finishing write tid 7 to mira0437250-113 2014-07-01 16:56:39,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 113 v 427 (ObjNum 719 snap 97 seq_num 719) dirty exists 2014-07-01 16:56:39,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: finishing write tid 4 to mira0437250-267 2014-07-01 16:56:39,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 334 (ObjNum 718 snap 97 seq_num 718) dirty exists 2014-07-01 16:56:39,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1100: done (4 left) 2014-07-01 16:56:39,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1101: done (3 left) 2014-07-01 16:56:39,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1102: done (2 left) 2014-07-01 16:56:39,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1104: done (1 left) 2014-07-01 16:56:39,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1106: done (0 left) 2014-07-01 16:56:39,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: snap_remove snap 96 2014-07-01 16:56:39,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1107: done (0 left) 2014-07-01 16:56:39,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: write oid 301 current snap is 97 2014-07-01 16:56:39,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: seq_num 721 ranges {611364=723378,1411363=1} 2014-07-01 16:56:39,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: writing mira0437250-301 from 611364 to 1334742 tid 1 2014-07-01 16:56:39,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: writing mira0437250-301 from 1411363 to 1411364 tid 2 2014-07-01 16:56:39,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: read oid 147 snap -1 2014-07-01 16:56:39,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: write oid 337 current snap is 97 2014-07-01 16:56:39,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: seq_num 722 ranges {423970=695183,1623969=1} 2014-07-01 16:56:39,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0437250-337 from 423970 to 1119153 tid 1 2014-07-01 16:56:39,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: writing mira0437250-337 from 1623969 to 1623970 tid 2 2014-07-01 16:56:39,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1111: delete oid 182 current snap is 97 2014-07-01 16:56:39,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: expect (ObjNum 570 snap 27 seq_num 570) 2014-07-01 16:56:40,371.371 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1] out_osds: [4] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:56:40,371.371 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:56:40,371.371 INFO:teuthology.task.thrashosds.thrasher:Adding osd 4 2014-07-01 16:56:40,372.372 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 4' 2014-07-01 16:56:40,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: finishing write tid 1 to mira0437250-301 2014-07-01 16:56:40,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: finishing write tid 2 to mira0437250-301 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: finishing write tid 4 to mira0437250-301 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 1 to mira0437250-337 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 2 to mira0437250-337 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 4 to mira0437250-337 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: finishing write tid 5 to mira0437250-301 2014-07-01 16:56:40,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 325 (ObjNum 721 snap 97 seq_num 721) dirty exists 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: finishing write tid 5 to mira0437250-337 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 332 (ObjNum 722 snap 97 seq_num 722) dirty exists 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1108: done (3 left) 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1109: done (2 left) 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1110: done (1 left) 2014-07-01 16:56:40,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:1111: done (0 left) 2014-07-01 16:56:40,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: write oid 42 current snap is 97 2014-07-01 16:56:40,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: seq_num 724 ranges {464029=790215,1869425=194604} 2014-07-01 16:56:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: writing mira0437250-42 from 464029 to 1254244 tid 1 2014-07-01 16:56:40,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: writing mira0437250-42 from 1869425 to 2064029 tid 2 2014-07-01 16:56:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: write oid 220 current snap is 97 2014-07-01 16:56:40,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: seq_num 725 ranges {408859=798116,1868230=589092,2808858=1} 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: writing mira0437250-220 from 408859 to 1206975 tid 1 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: writing mira0437250-220 from 1868230 to 2457322 tid 2 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: writing mira0437250-220 from 2808858 to 2808859 tid 3 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: snap_create 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: finishing write tid 1 to mira0437250-42 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: finishing write tid 2 to mira0437250-42 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: finishing write tid 4 to mira0437250-42 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: finishing write tid 5 to mira0437250-42 2014-07-01 16:56:40,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 375 (ObjNum 724 snap 97 seq_num 724) dirty exists 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: finishing write tid 1 to mira0437250-220 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: finishing write tid 2 to mira0437250-220 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: finishing write tid 3 to mira0437250-220 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: finishing write tid 5 to mira0437250-220 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: finishing write tid 6 to mira0437250-220 2014-07-01 16:56:40,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 190 (ObjNum 725 snap 97 seq_num 725) dirty exists 2014-07-01 16:56:41,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1112: done (2 left) 2014-07-01 16:56:41,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1113: done (1 left) 2014-07-01 16:56:41,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1114: done (0 left) 2014-07-01 16:56:41,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1115: delete oid 404 current snap is 98 2014-07-01 16:56:41,650.650 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.4. 2014-07-01 16:56:41,662.662 INFO:teuthology.task.thrashosds.thrasher:Added osd 4 2014-07-01 16:56:41,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1115: done (0 left) 2014-07-01 16:56:41,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: write oid 444 current snap is 98 2014-07-01 16:56:41,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: seq_num 727 ranges {564247=493148,1775726=480906,3045550=546881,3764246=1} 2014-07-01 16:56:41,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: writing mira0437250-444 from 564247 to 1057395 tid 1 2014-07-01 16:56:41,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: writing mira0437250-444 from 1775726 to 2256632 tid 2 2014-07-01 16:56:41,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: writing mira0437250-444 from 3045550 to 3592431 tid 3 2014-07-01 16:56:41,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: writing mira0437250-444 from 3764246 to 3764247 tid 4 2014-07-01 16:56:41,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: read oid 37 snap -1 2014-07-01 16:56:41,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 1 to mira0437250-444 2014-07-01 16:56:41,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 2 to mira0437250-444 2014-07-01 16:56:41,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: snap_remove snap 95 2014-07-01 16:56:41,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 3 to mira0437250-444 2014-07-01 16:56:41,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 4 to mira0437250-444 2014-07-01 16:56:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 6 to mira0437250-444 2014-07-01 16:56:41,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: finishing write tid 7 to mira0437250-444 2014-07-01 16:56:41,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 314 (ObjNum 727 snap 98 seq_num 727) dirty exists 2014-07-01 16:56:41,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: expect (ObjNum 571 snap 27 seq_num 571) 2014-07-01 16:56:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2747: expect (ObjNum 1253 snap 0 seq_num 1253) 2014-07-01 16:56:42,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:1116: done (2 left) 2014-07-01 16:56:42,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:1117: done (1 left) 2014-07-01 16:56:42,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1118: done (0 left) 2014-07-01 16:56:42,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:1119: snap_remove snap 97 2014-07-01 16:56:43,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:1119: done (0 left) 2014-07-01 16:56:43,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: write oid 186 current snap is 98 2014-07-01 16:56:43,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: seq_num 728 ranges {509075=1} 2014-07-01 16:56:43,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: writing mira0437250-186 from 509075 to 509076 tid 1 2014-07-01 16:56:43,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: write oid 221 current snap is 98 2014-07-01 16:56:43,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: seq_num 729 ranges {204305=1} 2014-07-01 16:56:43,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: writing mira0437250-221 from 204305 to 204306 tid 1 2014-07-01 16:56:43,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: read oid 381 snap -1 2014-07-01 16:56:43,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: write oid 198 current snap is 98 2014-07-01 16:56:43,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: seq_num 730 ranges {549136=400000} 2014-07-01 16:56:43,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: writing mira0437250-198 from 549136 to 949136 tid 1 2014-07-01 16:56:43,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: snap_create 2014-07-01 16:56:43,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: expect (ObjNum 380 snap 0 seq_num 380) 2014-07-01 16:56:43,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:2748: expect (ObjNum 1529 snap 0 seq_num 1529) 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: finishing write tid 1 to mira0437250-198 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: finishing write tid 3 to mira0437250-198 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: finishing write tid 4 to mira0437250-198 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 331 (ObjNum 730 snap 98 seq_num 730) dirty exists 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 1 to mira0437250-186 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 3 to mira0437250-186 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: finishing write tid 4 to mira0437250-186 2014-07-01 16:56:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 330 (ObjNum 728 snap 98 seq_num 728) dirty exists 2014-07-01 16:56:44,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:1120: done (4 left) 2014-07-01 16:56:44,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1122: done (3 left) 2014-07-01 16:56:44,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:1123: done (2 left) 2014-07-01 16:56:44,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1124: done (1 left) 2014-07-01 16:56:44,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: read oid 417 snap 53 2014-07-01 16:56:44,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: rollback oid 366 current snap is 99 2014-07-01 16:56:44,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 366 to 67 2014-07-01 16:56:44,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: read oid 296 snap -1 2014-07-01 16:56:44,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1128: snap_remove snap 72 2014-07-01 16:56:44,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: expect (ObjNum 295 snap 0 seq_num 295) 2014-07-01 16:56:44,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: expect (ObjNum 416 snap 0 seq_num 416) 2014-07-01 16:56:45,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: done (6 left) 2014-07-01 16:56:45,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: done (5 left) 2014-07-01 16:56:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: done (4 left) 2014-07-01 16:56:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: done (3 left) 2014-07-01 16:56:45,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:2746: done (2 left) 2014-07-01 16:56:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2747: done (1 left) 2014-07-01 16:56:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2748: done (0 left) 2014-07-01 16:56:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2749: read oid 457 snap -1 2014-07-01 16:56:45,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 246 current snap is 0 2014-07-01 16:56:45,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: seq_num 1733 ranges {1458176=438272,1896448=593920,2490368=655360,3145728=356352} 2014-07-01 16:56:45,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0435601-246 from 1458176 to 1896448 tid 1 2014-07-01 16:56:45,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0435601-246 from 1896448 to 2490368 tid 2 2014-07-01 16:56:45,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0435601-246 from 2490368 to 3145728 tid 3 2014-07-01 16:56:45,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0435601-246 from 3145728 to 3502080 tid 4 2014-07-01 16:56:45,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: read oid 491 snap -1 2014-07-01 16:56:45,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2749: expect (ObjNum 1145 snap 0 seq_num 1145) 2014-07-01 16:56:45,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 354 (ObjNum 624 snap 52 seq_num 624) dirty exists 2014-07-01 16:56:45,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1125: done (4 left) 2014-07-01 16:56:45,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1126: done (3 left) 2014-07-01 16:56:45,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1127: done (2 left) 2014-07-01 16:56:45,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1128: done (1 left) 2014-07-01 16:56:45,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: write oid 102 current snap is 99 2014-07-01 16:56:45,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: seq_num 731 ranges {610318=416462,1741887=468431} 2014-07-01 16:56:45,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: writing mira0437250-102 from 610318 to 1026780 tid 1 2014-07-01 16:56:45,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: writing mira0437250-102 from 1741887 to 2210318 tid 2 2014-07-01 16:56:45,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: write oid 454 current snap is 99 2014-07-01 16:56:45,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: seq_num 732 ranges {705547=789290,1905546=1} 2014-07-01 16:56:45,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: writing mira0437250-454 from 705547 to 1494837 tid 1 2014-07-01 16:56:45,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: writing mira0437250-454 from 1905546 to 1905547 tid 2 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: read oid 277 snap 91 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: read oid 200 snap 71 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1133: rollback oid 346 current snap is 99 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 346 to 64 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: snap_create 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: finishing write tid 1 to mira0437250-102 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: finishing write tid 2 to mira0437250-102 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: finishing write tid 4 to mira0437250-102 2014-07-01 16:56:45,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: expect (ObjNum 199 snap 0 seq_num 199) 2014-07-01 16:56:46,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: finishing write tid 5 to mira0437250-102 2014-07-01 16:56:46,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 338 (ObjNum 731 snap 99 seq_num 731) dirty exists 2014-07-01 16:56:46,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 328 (ObjNum 345 snap 0 seq_num 345) dirty exists 2014-07-01 16:56:46,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: expect (ObjNum 276 snap 0 seq_num 276) 2014-07-01 16:56:46,663.663 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1, 4] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:56:46,664.664 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:56:46,664.664 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 16:56:46,664.664 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:56:46,664.664 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:56:46,665.665 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 16:56:47,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:1129: done (6 left) 2014-07-01 16:56:47,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:1131: done (5 left) 2014-07-01 16:56:47,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1132: done (4 left) 2014-07-01 16:56:47,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1133: done (3 left) 2014-07-01 16:56:47,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1134: done (2 left) 2014-07-01 16:56:47,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:1135: delete oid 32 current snap is 100 2014-07-01 16:56:47,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: expect (ObjNum 963 snap 0 seq_num 963) 2014-07-01 16:56:47,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: finishing write tid 1 to mira0437250-454 2014-07-01 16:56:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: finishing write tid 2 to mira0437250-454 2014-07-01 16:56:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: finishing write tid 4 to mira0437250-454 2014-07-01 16:56:49,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 1 to mira0435601-246 2014-07-01 16:56:49,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 2 to mira0435601-246 2014-07-01 16:56:49,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 3 to mira0435601-246 2014-07-01 16:56:49,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 4 to mira0435601-246 2014-07-01 16:56:49,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 6 to mira0435601-246 2014-07-01 16:56:49,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 7 to mira0435601-246 2014-07-01 16:56:49,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 148 (ObjNum 1733 snap 0 seq_num 1733) dirty exists 2014-07-01 16:56:49,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:2749: done (2 left) 2014-07-01 16:56:49,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: done (1 left) 2014-07-01 16:56:49,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: done (0 left) 2014-07-01 16:56:49,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 272 current snap is 0 2014-07-01 16:56:49,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: seq_num 1734 ranges {1433600=466944,1900544=401408} 2014-07-01 16:56:49,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: writing mira0435601-272 from 1433600 to 1900544 tid 1 2014-07-01 16:56:49,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: writing mira0435601-272 from 1900544 to 2301952 tid 2 2014-07-01 16:56:49,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2753: delete oid 374 current snap is 0 2014-07-01 16:56:49,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: finishing write tid 1 to mira0435601-272 2014-07-01 16:56:49,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: finishing write tid 2 to mira0435601-272 2014-07-01 16:56:49,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: finishing write tid 4 to mira0435601-272 2014-07-01 16:56:49,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: finishing write tid 5 to mira0435601-272 2014-07-01 16:56:49,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 150 (ObjNum 1734 snap 0 seq_num 1734) dirty exists 2014-07-01 16:56:49,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1135: done (2 left) 2014-07-01 16:56:49,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: write oid 10 current snap is 100 2014-07-01 16:56:49,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: seq_num 734 ranges {594183=768995,1897787=757744,3277808=116375} 2014-07-01 16:56:49,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: writing mira0437250-10 from 594183 to 1363178 tid 1 2014-07-01 16:56:49,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: writing mira0437250-10 from 1897787 to 2655531 tid 2 2014-07-01 16:56:49,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: writing mira0437250-10 from 3277808 to 3394183 tid 3 2014-07-01 16:56:49,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: snap_remove snap 35 2014-07-01 16:56:49,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: finishing write tid 1 to mira0437250-10 2014-07-01 16:56:49,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 1 to mira0437250-221 2014-07-01 16:56:49,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: finishing write tid 2 to mira0437250-10 2014-07-01 16:56:49,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 3 to mira0437250-221 2014-07-01 16:56:49,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: finishing write tid 3 to mira0437250-10 2014-07-01 16:56:49,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: finishing write tid 5 to mira0437250-10 2014-07-01 16:56:49,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: finishing write tid 4 to mira0437250-221 2014-07-01 16:56:49,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 440 (ObjNum 729 snap 98 seq_num 729) dirty exists 2014-07-01 16:56:49,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: finishing write tid 6 to mira0437250-10 2014-07-01 16:56:49,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 10 v 186 (ObjNum 734 snap 100 seq_num 734) dirty exists 2014-07-01 16:56:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1121: done (3 left) 2014-07-01 16:56:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1136: done (2 left) 2014-07-01 16:56:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1137: done (1 left) 2014-07-01 16:56:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: write oid 411 current snap is 100 2014-07-01 16:56:50,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: seq_num 735 ranges {405764=492223,1478670=579364,2732281=511989,3605763=1} 2014-07-01 16:56:50,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: writing mira0437250-411 from 405764 to 897987 tid 1 2014-07-01 16:56:50,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: writing mira0437250-411 from 1478670 to 2058034 tid 2 2014-07-01 16:56:50,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: writing mira0437250-411 from 2732281 to 3244270 tid 3 2014-07-01 16:56:50,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: writing mira0437250-411 from 3605763 to 3605764 tid 4 2014-07-01 16:56:50,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:1139: snap_remove snap 75 2014-07-01 16:56:50,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: done (1 left) 2014-07-01 16:56:50,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2753: done (0 left) 2014-07-01 16:56:50,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: read oid 200 snap -1 2014-07-01 16:56:50,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: read oid 356 snap -1 2014-07-01 16:56:50,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 16:56:50,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: seq_num 1736 ranges {6168576=659456,6828032=589824,7417856=212992} 2014-07-01 16:56:50,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: writing mira0435601-313 from 6168576 to 6828032 tid 1 2014-07-01 16:56:50,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: writing mira0435601-313 from 6828032 to 7417856 tid 2 2014-07-01 16:56:50,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: writing mira0435601-313 from 7417856 to 7630848 tid 3 2014-07-01 16:56:50,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2757: read oid 427 snap -1 2014-07-01 16:56:50,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2758: read oid 11 snap -1 2014-07-01 16:56:50,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2759: delete oid 201 current snap is 0 2014-07-01 16:56:50,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: expect (ObjNum 1669 snap 0 seq_num 1669) 2014-07-01 16:56:51,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1139: done (2 left) 2014-07-01 16:56:51,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: rollback oid 382 current snap is 100 2014-07-01 16:56:51,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 382 to 87 2014-07-01 16:56:51,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1141: snap_create 2014-07-01 16:56:52,112.112 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 2, 0, 5, 1, 4] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:56:52,113.113 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:56:52,113.113 INFO:teuthology.task.thrashosds.thrasher:Removing osd 2, in_osds are: [3, 2, 0, 5, 1, 4] 2014-07-01 16:56:52,114.114 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 2' 2014-07-01 16:56:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1141: done (3 left) 2014-07-01 16:56:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1142: rollback oid 195 current snap is 101 2014-07-01 16:56:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 195 to 64 2014-07-01 16:56:52,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1143: read oid 210 snap -1 2014-07-01 16:56:52,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: write oid 237 current snap is 101 2014-07-01 16:56:52,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: seq_num 736 ranges {350593=1} 2014-07-01 16:56:52,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: writing mira0437250-237 from 350593 to 350594 tid 1 2014-07-01 16:56:52,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:1145: snap_create 2014-07-01 16:56:52,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 1 to mira0437250-411 2014-07-01 16:56:52,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 204 (ObjNum 194 snap 0 seq_num 194) dirty exists 2014-07-01 16:56:52,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: finishing write tid 5 to mira0437250-454 2014-07-01 16:56:52,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 283 (ObjNum 732 snap 99 seq_num 732) dirty exists 2014-07-01 16:56:52,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 2 to mira0437250-411 2014-07-01 16:56:52,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 3 to mira0437250-411 2014-07-01 16:56:52,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 4 to mira0437250-411 2014-07-01 16:56:52,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 6 to mira0437250-411 2014-07-01 16:56:52,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 328 (ObjNum 381 snap 0 seq_num 381) dirty exists 2014-07-01 16:56:52,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: finishing write tid 1 to mira0437250-237 2014-07-01 16:56:52,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: finishing write tid 3 to mira0437250-237 2014-07-01 16:56:52,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: finishing write tid 4 to mira0437250-237 2014-07-01 16:56:52,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 342 (ObjNum 736 snap 101 seq_num 736) dirty exists 2014-07-01 16:56:52,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: finishing write tid 7 to mira0437250-411 2014-07-01 16:56:52,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 294 (ObjNum 735 snap 100 seq_num 735) dirty exists 2014-07-01 16:56:53,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:1130: done (6 left) 2014-07-01 16:56:53,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1138: done (5 left) 2014-07-01 16:56:53,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1140: done (4 left) 2014-07-01 16:56:53,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1142: done (3 left) 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1143: done (2 left) 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1144: done (1 left) 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1145: done (0 left) 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: write oid 484 current snap is 102 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: seq_num 737 ranges {45823=1} 2014-07-01 16:56:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: writing mira0437250-484 from 45823 to 45824 tid 1 2014-07-01 16:56:53,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: snap_create 2014-07-01 16:56:53,157.157 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.2. 2014-07-01 16:56:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:1147: done (1 left) 2014-07-01 16:56:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1148: delete oid 33 current snap is 103 2014-07-01 16:56:55,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: finishing write tid 1 to mira0435601-313 2014-07-01 16:56:55,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: finishing write tid 2 to mira0435601-313 2014-07-01 16:56:55,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: expect (ObjNum 953 snap 0 seq_num 953) 2014-07-01 16:56:55,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 1 to mira0437250-484 2014-07-01 16:56:55,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 3 to mira0437250-484 2014-07-01 16:56:56,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: finishing write tid 4 to mira0437250-484 2014-07-01 16:56:56,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 189 (ObjNum 737 snap 102 seq_num 737) dirty exists 2014-07-01 16:56:57,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2757: expect (ObjNum 1464 snap 0 seq_num 1464) 2014-07-01 16:56:58,171.171 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 1, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:56:58,171.171 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:56:58,171.171 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_1 2014-07-01 16:56:58,171.171 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:56:59,005.005 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:56:59,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: finishing write tid 3 to mira0435601-313 2014-07-01 16:56:59,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: finishing write tid 5 to mira0435601-313 2014-07-01 16:56:59,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: finishing write tid 6 to mira0435601-313 2014-07-01 16:56:59,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 207 (ObjNum 1736 snap 0 seq_num 1736) dirty exists 2014-07-01 16:56:59,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2758: expect (ObjNum 1730 snap 0 seq_num 1730) 2014-07-01 16:56:59,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1146: done (1 left) 2014-07-01 16:56:59,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1148: done (0 left) 2014-07-01 16:56:59,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1149: rollback oid 21 current snap is 103 2014-07-01 16:56:59,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 21 to 58 2014-07-01 16:56:59,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: write oid 118 current snap is 103 2014-07-01 16:56:59,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: seq_num 739 ranges {734459=567985,1534458=1} 2014-07-01 16:56:59,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: writing mira0437250-118 from 734459 to 1302444 tid 1 2014-07-01 16:56:59,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: writing mira0437250-118 from 1534458 to 1534459 tid 2 2014-07-01 16:56:59,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:1151: delete oid 423 current snap is 103 2014-07-01 16:56:59,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 354 (ObjNum 20 snap 0 seq_num 20) dirty exists 2014-07-01 16:56:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1149: done (2 left) 2014-07-01 16:56:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1151: done (1 left) 2014-07-01 16:56:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1152: rollback oid 498 current snap is 103 2014-07-01 16:56:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 498 to 99 2014-07-01 16:56:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: snap_create 2014-07-01 16:56:59,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 341 (ObjNum 517 snap 10 seq_num 517) dirty exists 2014-07-01 16:57:00,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1152: done (2 left) 2014-07-01 16:57:00,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1153: done (1 left) 2014-07-01 16:57:00,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: read oid 193 snap -1 2014-07-01 16:57:00,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1155: rollback oid 499 current snap is 104 2014-07-01 16:57:00,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 499 to 85 2014-07-01 16:57:00,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:1156: snap_create 2014-07-01 16:57:00,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: expect (ObjNum 192 snap 0 seq_num 192) 2014-07-01 16:57:00,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 499 v 0 (ObjNum 17070280 snap 0 seq_num 41991040) dirty dne 2014-07-01 16:57:01,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:1154: done (3 left) 2014-07-01 16:57:01,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1155: done (2 left) 2014-07-01 16:57:01,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1156: done (1 left) 2014-07-01 16:57:01,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: delete oid 104 current snap is 105 2014-07-01 16:57:01,782.782 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pgp_num 76' 2014-07-01 16:57:02,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1157: done (1 left) 2014-07-01 16:57:02,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: rollback oid 295 current snap is 105 2014-07-01 16:57:02,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 295 to 71 2014-07-01 16:57:02,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: read oid 493 snap 71 2014-07-01 16:57:02,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: read oid 421 snap -1 2014-07-01 16:57:02,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:1161: snap_remove snap 73 2014-07-01 16:57:02,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: expect (ObjNum 684 snap 83 seq_num 684) 2014-07-01 16:57:03,119.119 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pgp_num to 76 2014-07-01 16:57:03,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1160: done (4 left) 2014-07-01 16:57:03,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1161: done (3 left) 2014-07-01 16:57:03,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: snap_create 2014-07-01 16:57:04,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: done (5 left) 2014-07-01 16:57:04,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: done (4 left) 2014-07-01 16:57:04,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: done (3 left) 2014-07-01 16:57:04,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2757: done (2 left) 2014-07-01 16:57:04,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2758: done (1 left) 2014-07-01 16:57:04,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2759: done (0 left) 2014-07-01 16:57:04,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2760: read oid 387 snap -1 2014-07-01 16:57:04,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2761: read oid 281 snap -1 2014-07-01 16:57:04,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: read oid 454 snap -1 2014-07-01 16:57:04,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 164 current snap is 0 2014-07-01 16:57:04,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: seq_num 1738 ranges {2560000=159744} 2014-07-01 16:57:04,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: writing mira0435601-164 from 2560000 to 2719744 tid 1 2014-07-01 16:57:04,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 13 current snap is 0 2014-07-01 16:57:04,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: seq_num 1739 ranges {1441792=585728,2027520=585728,2613248=217088} 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: writing mira0435601-13 from 1441792 to 2027520 tid 1 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: writing mira0435601-13 from 2027520 to 2613248 tid 2 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: writing mira0435601-13 from 2613248 to 2830336 tid 3 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 152 current snap is 0 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: seq_num 1740 ranges {1736704=458752,2195456=569344,2764800=233472} 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: writing mira0435601-152 from 1736704 to 2195456 tid 1 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: writing mira0435601-152 from 2195456 to 2764800 tid 2 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: writing mira0435601-152 from 2764800 to 2998272 tid 3 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 188 current snap is 0 2014-07-01 16:57:04,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: seq_num 1741 ranges {2039808=782336,2822144=798720,3620864=405504} 2014-07-01 16:57:04,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: writing mira0435601-188 from 2039808 to 2822144 tid 1 2014-07-01 16:57:04,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: writing mira0435601-188 from 2822144 to 3620864 tid 2 2014-07-01 16:57:04,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: writing mira0435601-188 from 3620864 to 4026368 tid 3 2014-07-01 16:57:04,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2767: delete oid 273 current snap is 0 2014-07-01 16:57:04,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:1162: done (3 left) 2014-07-01 16:57:04,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: write oid 117 current snap is 106 2014-07-01 16:57:04,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: seq_num 742 ranges {435701=767046,1999709=457226,3116988=118713} 2014-07-01 16:57:04,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: writing mira0437250-117 from 435701 to 1202747 tid 1 2014-07-01 16:57:04,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2761: done (7 left) 2014-07-01 16:57:04,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2767: done (6 left) 2014-07-01 16:57:04,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2768: delete oid 196 current snap is 0 2014-07-01 16:57:04,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: writing mira0437250-117 from 1999709 to 2456935 tid 2 2014-07-01 16:57:04,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: writing mira0437250-117 from 3116988 to 3235701 tid 3 2014-07-01 16:57:04,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: snap_remove snap 94 2014-07-01 16:57:04,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2768: done (6 left) 2014-07-01 16:57:04,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 92 current snap is 0 2014-07-01 16:57:04,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: seq_num 1744 ranges {0=552960} 2014-07-01 16:57:04,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: writing mira0435601-92 from 0 to 552960 tid 1 2014-07-01 16:57:04,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: read oid 197 snap -1 2014-07-01 16:57:04,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: finishing write tid 1 to mira0435601-164 2014-07-01 16:57:04,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2771: delete oid 219 current snap is 0 2014-07-01 16:57:04,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: finishing write tid 3 to mira0435601-164 2014-07-01 16:57:04,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 1 to mira0437250-118 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 2 to mira0437250-118 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 4 to mira0437250-118 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: finishing write tid 5 to mira0437250-118 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 278 (ObjNum 739 snap 103 seq_num 739) dirty exists 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: finishing write tid 1 to mira0437250-117 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: finishing write tid 2 to mira0437250-117 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: finishing write tid 3 to mira0437250-117 2014-07-01 16:57:04,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: finishing write tid 5 to mira0437250-117 2014-07-01 16:57:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: finishing write tid 1 to mira0435601-92 2014-07-01 16:57:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: finishing write tid 4 to mira0435601-164 2014-07-01 16:57:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 274 (ObjNum 1738 snap 0 seq_num 1738) dirty exists 2014-07-01 16:57:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: finishing write tid 3 to mira0435601-92 2014-07-01 16:57:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: expect (ObjNum 1605 snap 0 seq_num 1605) 2014-07-01 16:57:04,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 328 (ObjNum 626 snap 53 seq_num 626) dirty exists 2014-07-01 16:57:04,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: expect (ObjNum 530 snap 13 seq_num 530) 2014-07-01 16:57:06,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: finishing write tid 6 to mira0437250-117 2014-07-01 16:57:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 117 v 202 (ObjNum 742 snap 106 seq_num 742) dirty exists 2014-07-01 16:57:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: finishing write tid 1 to mira0435601-188 2014-07-01 16:57:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: finishing write tid 4 to mira0435601-92 2014-07-01 16:57:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 254 (ObjNum 1744 snap 0 seq_num 1744) dirty exists 2014-07-01 16:57:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: finishing write tid 2 to mira0435601-188 2014-07-01 16:57:06,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2760: expect (ObjNum 1449 snap 0 seq_num 1449) 2014-07-01 16:57:06,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1150: done (4 left) 2014-07-01 16:57:06,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:1158: done (3 left) 2014-07-01 16:57:06,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:1159: done (2 left) 2014-07-01 16:57:06,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:1163: done (1 left) 2014-07-01 16:57:06,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1164: done (0 left) 2014-07-01 16:57:06,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1165: snap_create 2014-07-01 16:57:06,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: expect (ObjNum 453 snap 0 seq_num 453) 2014-07-01 16:57:07,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1165: done (0 left) 2014-07-01 16:57:07,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1166: snap_remove snap 56 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: finishing write tid 3 to mira0435601-188 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: finishing write tid 5 to mira0435601-188 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: finishing write tid 6 to mira0435601-188 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 151 (ObjNum 1741 snap 0 seq_num 1741) dirty exists 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: finishing write tid 1 to mira0435601-152 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: finishing write tid 2 to mira0435601-152 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: finishing write tid 3 to mira0435601-152 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: finishing write tid 5 to mira0435601-152 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: finishing write tid 6 to mira0435601-152 2014-07-01 16:57:07,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 154 (ObjNum 1740 snap 0 seq_num 1740) dirty exists 2014-07-01 16:57:08,135.135 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 1, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:08,135.135 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:08,135.135 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 4 to 0.307835167278 2014-07-01 16:57:08,136.136 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 0.307835167278' 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1166: done (0 left) 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1167: rollback oid 489 current snap is 107 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 489 to 51 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: read oid 265 snap -1 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: read oid 244 snap -1 2014-07-01 16:57:08,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: snap_create 2014-07-01 16:57:08,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: expect (ObjNum 614 snap 47 seq_num 614) 2014-07-01 16:57:08,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: finishing write tid 1 to mira0435601-13 2014-07-01 16:57:08,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: finishing write tid 2 to mira0435601-13 2014-07-01 16:57:08,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: finishing write tid 3 to mira0435601-13 2014-07-01 16:57:08,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: finishing write tid 5 to mira0435601-13 2014-07-01 16:57:08,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: finishing write tid 6 to mira0435601-13 2014-07-01 16:57:08,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 218 (ObjNum 1739 snap 0 seq_num 1739) dirty exists 2014-07-01 16:57:08,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: expect (ObjNum 264 snap 0 seq_num 264) 2014-07-01 16:57:09,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 523 (ObjNum 488 snap 0 seq_num 488) dirty exists 2014-07-01 16:57:09,518.518 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 0.307835 (8201742) 2014-07-01 16:57:10,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1167: done (3 left) 2014-07-01 16:57:10,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1168: done (2 left) 2014-07-01 16:57:10,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:1169: done (1 left) 2014-07-01 16:57:10,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:1170: done (0 left) 2014-07-01 16:57:10,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: write oid 40 current snap is 108 2014-07-01 16:57:10,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: seq_num 743 ranges {531954=491298,1466288=678845,2822084=109870} 2014-07-01 16:57:10,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: writing mira0437250-40 from 531954 to 1023252 tid 1 2014-07-01 16:57:11,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: writing mira0437250-40 from 1466288 to 2145133 tid 2 2014-07-01 16:57:11,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: writing mira0437250-40 from 2822084 to 2931954 tid 3 2014-07-01 16:57:11,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: read oid 290 snap 100 2014-07-01 16:57:11,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1173: rollback oid 224 current snap is 108 2014-07-01 16:57:11,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 224 to 105 2014-07-01 16:57:11,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: write oid 402 current snap is 108 2014-07-01 16:57:11,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: seq_num 744 ranges {191087=1} 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: writing mira0437250-402 from 191087 to 191088 tid 1 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: rollback oid 205 current snap is 108 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 205 to 49 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: read oid 103 snap -1 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: read oid 73 snap -1 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: write oid 471 current snap is 108 2014-07-01 16:57:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: seq_num 745 ranges {535918=400000} 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: writing mira0437250-471 from 535918 to 935918 tid 1 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: delete oid 158 current snap is 108 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: expect (ObjNum 102 snap 0 seq_num 102) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2760: done (8 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: done (7 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: done (6 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: done (5 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: done (4 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: done (3 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: done (2 left) 2014-07-01 16:57:11,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: done (1 left) 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2771: done (0 left) 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2772: read oid 10 snap -1 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 117 current snap is 0 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: seq_num 1746 ranges {0=98304} 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: writing mira0435601-117 from 0 to 98304 tid 1 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: read oid 369 snap -1 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 88 current snap is 0 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: seq_num 1747 ranges {708608=770048,1478656=790528,2269184=696320,2965504=118784} 2014-07-01 16:57:11,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: writing mira0435601-88 from 708608 to 1478656 tid 1 2014-07-01 16:57:11,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: writing mira0435601-88 from 1478656 to 2269184 tid 2 2014-07-01 16:57:11,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: writing mira0435601-88 from 2269184 to 2965504 tid 3 2014-07-01 16:57:11,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: writing mira0435601-88 from 2965504 to 3084288 tid 4 2014-07-01 16:57:11,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 16:57:11,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: seq_num 1748 ranges {5451776=798720,6250496=401408} 2014-07-01 16:57:11,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: writing mira0435601-492 from 5451776 to 6250496 tid 1 2014-07-01 16:57:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: writing mira0435601-492 from 6250496 to 6651904 tid 2 2014-07-01 16:57:11,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 362 current snap is 0 2014-07-01 16:57:11,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: seq_num 1749 ranges {4939776=24576} 2014-07-01 16:57:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: writing mira0435601-362 from 4939776 to 4964352 tid 1 2014-07-01 16:57:11,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 193 current snap is 0 2014-07-01 16:57:11,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: seq_num 1750 ranges {6463488=589824,7053312=540672,7593984=663552} 2014-07-01 16:57:11,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: writing mira0435601-193 from 6463488 to 7053312 tid 1 2014-07-01 16:57:11,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: writing mira0435601-193 from 7053312 to 7593984 tid 2 2014-07-01 16:57:11,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: writing mira0435601-193 from 7593984 to 8257536 tid 3 2014-07-01 16:57:11,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 227 current snap is 0 2014-07-01 16:57:11,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: seq_num 1751 ranges {3862528=618496} 2014-07-01 16:57:11,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: writing mira0435601-227 from 3862528 to 4481024 tid 1 2014-07-01 16:57:11,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 280 current snap is 0 2014-07-01 16:57:11,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: seq_num 1752 ranges {4263936=647168,4911104=757760,5668864=446464} 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: writing mira0435601-280 from 4263936 to 4911104 tid 1 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: writing mira0435601-280 from 4911104 to 5668864 tid 2 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: writing mira0435601-280 from 5668864 to 6115328 tid 3 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2781: read oid 38 snap -1 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: expect (ObjNum 1297 snap 0 seq_num 1297) 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 440 (ObjNum 655 snap 73 seq_num 655) dirty exists 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: finishing write tid 1 to mira0437250-471 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: finishing write tid 3 to mira0437250-471 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: finishing write tid 4 to mira0437250-471 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 313 (ObjNum 745 snap 108 seq_num 745) dirty exists 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 1 to mira0437250-40 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 2 to mira0437250-40 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 3 to mira0437250-40 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 5 to mira0437250-40 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: finishing write tid 6 to mira0437250-40 2014-07-01 16:57:11,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 452 (ObjNum 743 snap 108 seq_num 743) dirty exists 2014-07-01 16:57:11,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: finishing write tid 1 to mira0437250-402 2014-07-01 16:57:11,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: finishing write tid 3 to mira0437250-402 2014-07-01 16:57:11,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: finishing write tid 4 to mira0437250-402 2014-07-01 16:57:11,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 371 (ObjNum 744 snap 108 seq_num 744) dirty exists 2014-07-01 16:57:11,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: expect (ObjNum 72 snap 0 seq_num 72) 2014-07-01 16:57:11,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2781: expect (ObjNum 1094 snap 0 seq_num 1094) 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 1 to mira0435601-362 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 3 to mira0435601-362 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 1 to mira0435601-88 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 1 to mira0435601-117 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 3 to mira0435601-117 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 2 to mira0435601-88 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 3 to mira0435601-88 2014-07-01 16:57:12,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 4 to mira0435601-88 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 6 to mira0435601-88 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 7 to mira0435601-88 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 231 (ObjNum 1747 snap 0 seq_num 1747) dirty exists 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 4 to mira0435601-362 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 272 (ObjNum 1749 snap 0 seq_num 1749) dirty exists 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 4 to mira0435601-117 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 117 v 160 (ObjNum 1746 snap 0 seq_num 1746) dirty exists 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: finishing write tid 1 to mira0435601-193 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2772: done (9 left) 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: done (8 left) 2014-07-01 16:57:12,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: done (7 left) 2014-07-01 16:57:12,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: done (6 left) 2014-07-01 16:57:12,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: done (5 left) 2014-07-01 16:57:12,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2781: done (4 left) 2014-07-01 16:57:12,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 399 current snap is 0 2014-07-01 16:57:12,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: seq_num 1753 ranges {0=360448} 2014-07-01 16:57:12,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: writing mira0435601-399 from 0 to 360448 tid 1 2014-07-01 16:57:12,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:57:12,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: seq_num 1754 ranges {6832128=36864} 2014-07-01 16:57:13,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 205 v 352 (ObjNum 204 snap 0 seq_num 204) dirty exists 2014-07-01 16:57:13,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 16:57:13,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: writing mira0435601-200 from 6832128 to 6868992 tid 1 2014-07-01 16:57:13,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:57:13,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: seq_num 1755 ranges {5230592=466944,5697536=753664,6451200=49152} 2014-07-01 16:57:13,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: writing mira0435601-147 from 5230592 to 5697536 tid 1 2014-07-01 16:57:13,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: writing mira0435601-147 from 5697536 to 6451200 tid 2 2014-07-01 16:57:13,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: writing mira0435601-147 from 6451200 to 6500352 tid 3 2014-07-01 16:57:13,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: read oid 127 snap -1 2014-07-01 16:57:13,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: finishing write tid 2 to mira0435601-193 2014-07-01 16:57:13,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: finishing write tid 3 to mira0435601-193 2014-07-01 16:57:13,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: finishing write tid 5 to mira0435601-193 2014-07-01 16:57:13,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: finishing write tid 1 to mira0435601-280 2014-07-01 16:57:13,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: finishing write tid 6 to mira0435601-193 2014-07-01 16:57:13,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 193 v 320 (ObjNum 1750 snap 0 seq_num 1750) dirty exists 2014-07-01 16:57:13,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: finishing write tid 2 to mira0435601-280 2014-07-01 16:57:13,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: finishing write tid 3 to mira0435601-280 2014-07-01 16:57:13,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: finishing write tid 5 to mira0435601-280 2014-07-01 16:57:13,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: finishing write tid 6 to mira0435601-280 2014-07-01 16:57:13,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 324 (ObjNum 1752 snap 0 seq_num 1752) dirty exists 2014-07-01 16:57:13,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: finishing write tid 1 to mira0435601-227 2014-07-01 16:57:13,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: finishing write tid 3 to mira0435601-227 2014-07-01 16:57:13,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: finishing write tid 4 to mira0435601-227 2014-07-01 16:57:13,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 212 (ObjNum 1751 snap 0 seq_num 1751) dirty exists 2014-07-01 16:57:13,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: done (7 left) 2014-07-01 16:57:13,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: done (6 left) 2014-07-01 16:57:13,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: done (5 left) 2014-07-01 16:57:13,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 317 current snap is 0 2014-07-01 16:57:13,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: seq_num 1756 ranges {1302528=581632,1884160=581632,2465792=712704,3178496=311296} 2014-07-01 16:57:13,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: writing mira0435601-317 from 1302528 to 1884160 tid 1 2014-07-01 16:57:13,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: writing mira0435601-317 from 1884160 to 2465792 tid 2 2014-07-01 16:57:13,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: writing mira0435601-317 from 2465792 to 3178496 tid 3 2014-07-01 16:57:13,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1171: done (8 left) 2014-07-01 16:57:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1172: done (7 left) 2014-07-01 16:57:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1173: done (6 left) 2014-07-01 16:57:13,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:1174: done (5 left) 2014-07-01 16:57:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1175: done (4 left) 2014-07-01 16:57:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1176: done (3 left) 2014-07-01 16:57:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1177: done (2 left) 2014-07-01 16:57:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1178: done (1 left) 2014-07-01 16:57:13,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1179: done (0 left) 2014-07-01 16:57:13,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:1180: snap_remove snap 105 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: writing mira0435601-317 from 3178496 to 3489792 tid 4 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 396 current snap is 0 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: seq_num 1757 ranges {3059712=610304,3670016=401408} 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: writing mira0435601-396 from 3059712 to 3670016 tid 1 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: writing mira0435601-396 from 3670016 to 4071424 tid 2 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 74 current snap is 0 2014-07-01 16:57:13,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: seq_num 1758 ranges {2510848=638976,3149824=749568,3899392=610304,4509696=245760} 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: writing mira0435601-74 from 2510848 to 3149824 tid 1 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: writing mira0435601-74 from 3149824 to 3899392 tid 2 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: writing mira0435601-74 from 3899392 to 4509696 tid 3 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: writing mira0435601-74 from 4509696 to 4755456 tid 4 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: read oid 447 snap -1 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2790: read oid 214 snap -1 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: read oid 495 snap -1 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2792: read oid 195 snap -1 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 264 current snap is 0 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: seq_num 1759 ranges {1589248=802816,2392064=577536,2969600=225280} 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1180: done (0 left) 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1181: rollback oid 338 current snap is 108 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 338 to 67 2014-07-01 16:57:13,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: snap_remove snap 99 2014-07-01 16:57:13,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: writing mira0435601-264 from 1589248 to 2392064 tid 1 2014-07-01 16:57:13,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: writing mira0435601-264 from 2392064 to 2969600 tid 2 2014-07-01 16:57:13,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: writing mira0435601-264 from 2969600 to 3194880 tid 3 2014-07-01 16:57:13,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: read oid 440 snap -1 2014-07-01 16:57:13,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: expect (ObjNum 1584 snap 0 seq_num 1584) 2014-07-01 16:57:14,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1182: done (1 left) 2014-07-01 16:57:14,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: snap_create 2014-07-01 16:57:14,521.521 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 1, 4] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:14,521.521 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:14,521.521 INFO:teuthology.task.thrashosds.thrasher:Adding osd 2 2014-07-01 16:57:14,521.521 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 2' 2014-07-01 16:57:14,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: done (13 left) 2014-07-01 16:57:14,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: read oid 436 snap -1 2014-07-01 16:57:14,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2796: delete oid 230 current snap is 0 2014-07-01 16:57:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1183: done (1 left) 2014-07-01 16:57:15,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1184: delete oid 78 current snap is 109 2014-07-01 16:57:15,142.142 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.2. 2014-07-01 16:57:15,159.159 INFO:teuthology.task.thrashosds.thrasher:Added osd 2 2014-07-01 16:57:16,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2796: done (14 left) 2014-07-01 16:57:16,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: delete oid 476 current snap is 0 2014-07-01 16:57:16,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 1 to mira0435601-399 2014-07-01 16:57:16,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 3 to mira0435601-399 2014-07-01 16:57:16,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: finishing write tid 1 to mira0435601-147 2014-07-01 16:57:16,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: finishing write tid 2 to mira0435601-147 2014-07-01 16:57:16,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: finishing write tid 3 to mira0435601-147 2014-07-01 16:57:16,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: finishing write tid 5 to mira0435601-147 2014-07-01 16:57:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 359 (ObjNum 337 snap 0 seq_num 337) dirty exists 2014-07-01 16:57:17,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: finishing write tid 1 to mira0435601-396 2014-07-01 16:57:17,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 4 to mira0435601-399 2014-07-01 16:57:17,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 399 v 162 (ObjNum 1753 snap 0 seq_num 1753) dirty exists 2014-07-01 16:57:17,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: finishing write tid 6 to mira0435601-147 2014-07-01 16:57:17,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 148 (ObjNum 1755 snap 0 seq_num 1755) dirty exists 2014-07-01 16:57:17,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: finishing write tid 2 to mira0435601-396 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1181: done (1 left) 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1184: done (0 left) 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1185: rollback oid 326 current snap is 109 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 326 to 108 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: rollback oid 480 current snap is 109 2014-07-01 16:57:17,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 480 to 71 2014-07-01 16:57:17,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1187: read oid 113 snap 53 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: read oid 342 snap 53 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: write oid 292 current snap is 109 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: seq_num 748 ranges {520807=537840,1812309=308498} 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 1 to mira0435601-317 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 1 to mira0435601-74 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 2 to mira0435601-317 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 3 to mira0435601-317 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 4 to mira0435601-317 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 6 to mira0435601-317 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: finishing write tid 4 to mira0435601-396 2014-07-01 16:57:17,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 2 to mira0435601-74 2014-07-01 16:57:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 3 to mira0435601-74 2014-07-01 16:57:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 4 to mira0435601-74 2014-07-01 16:57:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: finishing write tid 5 to mira0435601-396 2014-07-01 16:57:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 284 (ObjNum 1757 snap 0 seq_num 1757) dirty exists 2014-07-01 16:57:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: finishing write tid 7 to mira0435601-317 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 236 (ObjNum 1756 snap 0 seq_num 1756) dirty exists 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: writing mira0437250-292 from 520807 to 1058647 tid 1 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: writing mira0437250-292 from 1812309 to 2120807 tid 2 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1190: rollback oid 95 current snap is 109 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 95 to 100 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 502 (ObjNum 325 snap 0 seq_num 325) dirty exists 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 251 (ObjNum 479 snap 0 seq_num 479) dirty exists 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1185: done (5 left) 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1186: done (4 left) 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: write oid 208 current snap is 109 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: seq_num 749 ranges {732389=544716,2041769=290620} 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: writing mira0437250-208 from 732389 to 1277105 tid 1 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: writing mira0437250-208 from 2041769 to 2332389 tid 2 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: rollback oid 73 current snap is 109 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 73 to 49 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 290 (ObjNum 94 snap 0 seq_num 94) dirty exists 2014-07-01 16:57:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1190: done (5 left) 2014-07-01 16:57:17,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1193: snap_remove snap 58 2014-07-01 16:57:17,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:1187: expect (ObjNum 112 snap 0 seq_num 112) 2014-07-01 16:57:17,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 6 to mira0435601-74 2014-07-01 16:57:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: finishing write tid 1 to mira0435601-264 2014-07-01 16:57:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: finishing write tid 2 to mira0435601-264 2014-07-01 16:57:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: finishing write tid 3 to mira0435601-264 2014-07-01 16:57:17,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: finishing write tid 5 to mira0435601-264 2014-07-01 16:57:17,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: expect (ObjNum 1633 snap 0 seq_num 1633) 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 242 (ObjNum 72 snap 0 seq_num 72) dirty exists 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: finishing write tid 1 to mira0437250-292 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1187: done (5 left) 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1192: done (4 left) 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1193: done (3 left) 2014-07-01 16:57:18,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1194: read oid 407 snap -1 2014-07-01 16:57:18,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: finishing write tid 2 to mira0437250-292 2014-07-01 16:57:18,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: finishing write tid 4 to mira0437250-292 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: finishing write tid 5 to mira0437250-292 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 472 (ObjNum 748 snap 109 seq_num 748) dirty exists 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: finishing write tid 1 to mira0437250-208 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: finishing write tid 2 to mira0437250-208 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: finishing write tid 4 to mira0437250-208 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: finishing write tid 5 to mira0437250-208 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 410 (ObjNum 749 snap 109 seq_num 749) dirty exists 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1189: done (3 left) 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1191: done (2 left) 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: write oid 196 current snap is 109 2014-07-01 16:57:18,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: seq_num 750 ranges {428643=517544,1356925=556707,2028642=1} 2014-07-01 16:57:18,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: writing mira0437250-196 from 428643 to 946187 tid 1 2014-07-01 16:57:18,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: writing mira0437250-196 from 1356925 to 1913632 tid 2 2014-07-01 16:57:18,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: writing mira0437250-196 from 2028642 to 2028643 tid 3 2014-07-01 16:57:18,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: snap_remove snap 65 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 1 to mira0437250-196 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 2 to mira0437250-196 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 3 to mira0437250-196 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 5 to mira0437250-196 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: finishing write tid 6 to mira0437250-196 2014-07-01 16:57:18,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 356 (ObjNum 750 snap 109 seq_num 750) dirty exists 2014-07-01 16:57:18,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: expect (ObjNum 341 snap 0 seq_num 341) 2014-07-01 16:57:19,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1194: expect (ObjNum 406 snap 0 seq_num 406) 2014-07-01 16:57:20,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1188: done (3 left) 2014-07-01 16:57:20,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1194: done (2 left) 2014-07-01 16:57:20,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1195: done (1 left) 2014-07-01 16:57:20,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1196: done (0 left) 2014-07-01 16:57:20,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: write oid 417 current snap is 109 2014-07-01 16:57:20,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: seq_num 751 ranges {622048=489349,1684561=626903,2874768=657758,3822047=1} 2014-07-01 16:57:20,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: writing mira0437250-417 from 622048 to 1111397 tid 1 2014-07-01 16:57:20,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2790: expect (ObjNum 1441 snap 0 seq_num 1441) 2014-07-01 16:57:20,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: writing mira0437250-417 from 1684561 to 2311464 tid 2 2014-07-01 16:57:20,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: writing mira0437250-417 from 2874768 to 3532526 tid 3 2014-07-01 16:57:20,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: writing mira0437250-417 from 3822047 to 3822048 tid 4 2014-07-01 16:57:20,159.159 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 1, 4, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:20,159.159 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:20,159.159 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2014-07-01 16:57:20,159.159 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 0' 2014-07-01 16:57:20,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: snap_create 2014-07-01 16:57:20,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1198: done (1 left) 2014-07-01 16:57:20,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1199: snap_remove snap 55 2014-07-01 16:57:20,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 1 to mira0437250-417 2014-07-01 16:57:20,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 2 to mira0437250-417 2014-07-01 16:57:20,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 3 to mira0437250-417 2014-07-01 16:57:21,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 4 to mira0437250-417 2014-07-01 16:57:21,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 6 to mira0437250-417 2014-07-01 16:57:21,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: finishing write tid 7 to mira0437250-417 2014-07-01 16:57:21,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 304 (ObjNum 751 snap 109 seq_num 751) dirty exists 2014-07-01 16:57:21,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1197: done (1 left) 2014-07-01 16:57:21,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:1199: done (0 left) 2014-07-01 16:57:21,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: delete oid 489 current snap is 110 2014-07-01 16:57:21,435.435 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 0 (802) 2014-07-01 16:57:21,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1200: done (0 left) 2014-07-01 16:57:21,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: write oid 376 current snap is 110 2014-07-01 16:57:21,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: seq_num 753 ranges {528860=469053,1511801=517693,2512290=677867,3727346=1514} 2014-07-01 16:57:21,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: writing mira0437250-376 from 528860 to 997913 tid 1 2014-07-01 16:57:21,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: writing mira0437250-376 from 1511801 to 2029494 tid 2 2014-07-01 16:57:21,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: writing mira0437250-376 from 2512290 to 3190157 tid 3 2014-07-01 16:57:21,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: writing mira0437250-376 from 3727346 to 3728860 tid 4 2014-07-01 16:57:21,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: delete oid 386 current snap is 110 2014-07-01 16:57:21,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 1 to mira0437250-376 2014-07-01 16:57:21,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 2 to mira0437250-376 2014-07-01 16:57:21,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 3 to mira0437250-376 2014-07-01 16:57:21,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 4 to mira0437250-376 2014-07-01 16:57:21,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 6 to mira0437250-376 2014-07-01 16:57:21,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: finishing write tid 7 to mira0437250-376 2014-07-01 16:57:21,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 508 (ObjNum 753 snap 110 seq_num 753) dirty exists 2014-07-01 16:57:22,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1201: done (1 left) 2014-07-01 16:57:22,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1202: done (0 left) 2014-07-01 16:57:22,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: read oid 186 snap -1 2014-07-01 16:57:22,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:1204: rollback oid 128 current snap is 110 2014-07-01 16:57:22,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 128 to 109 2014-07-01 16:57:22,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: write oid 288 current snap is 110 2014-07-01 16:57:22,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: seq_num 755 ranges {417495=564086,1217494=1} 2014-07-01 16:57:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: writing mira0437250-288 from 417495 to 981581 tid 1 2014-07-01 16:57:22,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: writing mira0437250-288 from 1217494 to 1217495 tid 2 2014-07-01 16:57:22,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1206: snap_create 2014-07-01 16:57:22,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: expect (ObjNum 728 snap 98 seq_num 728) 2014-07-01 16:57:22,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 1 to mira0437250-288 2014-07-01 16:57:22,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 2 to mira0437250-288 2014-07-01 16:57:22,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 4 to mira0437250-288 2014-07-01 16:57:22,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: finishing write tid 5 to mira0437250-288 2014-07-01 16:57:22,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 339 (ObjNum 755 snap 110 seq_num 755) dirty exists 2014-07-01 16:57:23,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1203: done (3 left) 2014-07-01 16:57:23,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1205: done (2 left) 2014-07-01 16:57:23,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:1206: done (1 left) 2014-07-01 16:57:23,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: write oid 282 current snap is 111 2014-07-01 16:57:23,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: seq_num 756 ranges {513749=400000} 2014-07-01 16:57:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: writing mira0437250-282 from 513749 to 913749 tid 1 2014-07-01 16:57:23,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: write oid 195 current snap is 111 2014-07-01 16:57:23,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: seq_num 757 ranges {706131=543791,2028363=433546,2706130=1} 2014-07-01 16:57:23,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: writing mira0437250-195 from 706131 to 1249922 tid 1 2014-07-01 16:57:23,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: writing mira0437250-195 from 2028363 to 2461909 tid 2 2014-07-01 16:57:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: writing mira0437250-195 from 2706130 to 2706131 tid 3 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: write oid 151 current snap is 111 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: seq_num 758 ranges {518736=515595,1707422=503741,2911697=7039} 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0437250-151 from 518736 to 1034331 tid 1 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0437250-151 from 1707422 to 2211163 tid 2 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: writing mira0437250-151 from 2911697 to 2918736 tid 3 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: delete oid 214 current snap is 111 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 1 to mira0437250-282 2014-07-01 16:57:23,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 3 to mira0437250-282 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: finishing write tid 4 to mira0437250-282 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 483 (ObjNum 756 snap 111 seq_num 756) dirty exists 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 1 to mira0437250-195 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 2 to mira0437250-195 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 3 to mira0437250-195 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 5 to mira0437250-195 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: finishing write tid 6 to mira0437250-195 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 248 (ObjNum 757 snap 111 seq_num 757) dirty exists 2014-07-01 16:57:23,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 1 to mira0437250-151 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 2 to mira0437250-151 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1207: done (4 left) 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1208: done (3 left) 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1210: done (2 left) 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: write oid 65 current snap is 111 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: seq_num 760 ranges {406347=1} 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: writing mira0437250-65 from 406347 to 406348 tid 1 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: snap_create 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 3 to mira0437250-151 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 5 to mira0437250-151 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: finishing write tid 6 to mira0437250-151 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 595 (ObjNum 758 snap 111 seq_num 758) dirty exists 2014-07-01 16:57:23,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 1 to mira0437250-65 2014-07-01 16:57:23,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 3 to mira0437250-65 2014-07-01 16:57:23,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: finishing write tid 4 to mira0437250-65 2014-07-01 16:57:23,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 443 (ObjNum 760 snap 111 seq_num 760) dirty exists 2014-07-01 16:57:23,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: expect (ObjNum 1653 snap 0 seq_num 1653) 2014-07-01 16:57:24,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1209: done (3 left) 2014-07-01 16:57:24,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1211: done (2 left) 2014-07-01 16:57:24,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1212: done (1 left) 2014-07-01 16:57:24,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: delete oid 277 current snap is 112 2014-07-01 16:57:24,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1213: done (1 left) 2014-07-01 16:57:24,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: read oid 320 snap 110 2014-07-01 16:57:24,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: write oid 414 current snap is 112 2014-07-01 16:57:24,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: seq_num 762 ranges {562759=439932,1362758=1} 2014-07-01 16:57:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: writing mira0437250-414 from 562759 to 1002691 tid 1 2014-07-01 16:57:24,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: writing mira0437250-414 from 1362758 to 1362759 tid 2 2014-07-01 16:57:24,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: write oid 460 current snap is 112 2014-07-01 16:57:24,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: seq_num 763 ranges {659013=400000} 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: writing mira0437250-460 from 659013 to 1059013 tid 1 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1217: rollback oid 101 current snap is 112 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 101 to 80 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 1 to mira0437250-414 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 2 to mira0437250-414 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 4 to mira0437250-414 2014-07-01 16:57:24,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: expect (ObjNum 319 snap 0 seq_num 319) 2014-07-01 16:57:25,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: finishing write tid 5 to mira0437250-414 2014-07-01 16:57:25,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 538 (ObjNum 762 snap 112 seq_num 762) dirty exists 2014-07-01 16:57:25,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 378 (ObjNum 637 snap 62 seq_num 637) dirty exists 2014-07-01 16:57:25,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1214: done (4 left) 2014-07-01 16:57:25,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1215: done (3 left) 2014-07-01 16:57:25,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1217: done (2 left) 2014-07-01 16:57:25,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1218: delete oid 2 current snap is 112 2014-07-01 16:57:25,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: finishing write tid 1 to mira0437250-460 2014-07-01 16:57:25,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: finishing write tid 3 to mira0437250-460 2014-07-01 16:57:25,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 242 (ObjNum 127 snap 0 seq_num 127) dirty exists 2014-07-01 16:57:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1204: done (2 left) 2014-07-01 16:57:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:1218: done (1 left) 2014-07-01 16:57:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: write oid 279 current snap is 112 2014-07-01 16:57:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: seq_num 765 ranges {547649=542865,1579883=649379,2547648=1} 2014-07-01 16:57:25,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: writing mira0437250-279 from 547649 to 1090514 tid 1 2014-07-01 16:57:25,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: writing mira0437250-279 from 1579883 to 2229262 tid 2 2014-07-01 16:57:25,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: writing mira0437250-279 from 2547648 to 2547649 tid 3 2014-07-01 16:57:25,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1220: delete oid 225 current snap is 112 2014-07-01 16:57:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:1220: done (2 left) 2014-07-01 16:57:25,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:1221: snap_create 2014-07-01 16:57:25,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: finishing write tid 1 to mira0437250-279 2014-07-01 16:57:25,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: finishing write tid 2 to mira0437250-279 2014-07-01 16:57:25,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: finishing write tid 3 to mira0437250-279 2014-07-01 16:57:25,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: finishing write tid 5 to mira0437250-279 2014-07-01 16:57:25,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: finishing write tid 6 to mira0437250-279 2014-07-01 16:57:25,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 400 (ObjNum 765 snap 112 seq_num 765) dirty exists 2014-07-01 16:57:25,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: finishing write tid 4 to mira0437250-460 2014-07-01 16:57:25,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 272 (ObjNum 763 snap 112 seq_num 763) dirty exists 2014-07-01 16:57:26,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1216: done (2 left) 2014-07-01 16:57:26,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1219: done (1 left) 2014-07-01 16:57:26,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1221: done (0 left) 2014-07-01 16:57:26,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1222: rollback oid 273 current snap is 113 2014-07-01 16:57:26,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 273 to 67 2014-07-01 16:57:26,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: write oid 341 current snap is 113 2014-07-01 16:57:26,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: seq_num 767 ranges {704060=638922,1889427=424842,2918054=402502,3504059=1} 2014-07-01 16:57:26,443.443 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 1, 4, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:26,443.443 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:26,443.443 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [3, 0, 5, 1, 4, 2] 2014-07-01 16:57:26,443.443 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 16:57:26,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: writing mira0437250-341 from 704060 to 1342982 tid 1 2014-07-01 16:57:26,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: writing mira0437250-341 from 1889427 to 2314269 tid 2 2014-07-01 16:57:26,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: writing mira0437250-341 from 2918054 to 3320556 tid 3 2014-07-01 16:57:26,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: writing mira0437250-341 from 3504059 to 3504060 tid 4 2014-07-01 16:57:26,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1224: snap_create 2014-07-01 16:57:26,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 404 (ObjNum 272 snap 0 seq_num 272) dirty exists 2014-07-01 16:57:26,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 1 to mira0437250-341 2014-07-01 16:57:26,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 2 to mira0437250-341 2014-07-01 16:57:26,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 3 to mira0437250-341 2014-07-01 16:57:26,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 4 to mira0437250-341 2014-07-01 16:57:26,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 6 to mira0437250-341 2014-07-01 16:57:26,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: finishing write tid 7 to mira0437250-341 2014-07-01 16:57:26,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 526 (ObjNum 767 snap 113 seq_num 767) dirty exists 2014-07-01 16:57:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1222: done (2 left) 2014-07-01 16:57:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1223: done (1 left) 2014-07-01 16:57:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1224: done (0 left) 2014-07-01 16:57:27,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: snap_create 2014-07-01 16:57:27,472.472 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 16:57:28,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: finishing write tid 6 to mira0435601-264 2014-07-01 16:57:28,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 333 (ObjNum 1759 snap 0 seq_num 1759) dirty exists 2014-07-01 16:57:28,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: finishing write tid 7 to mira0435601-74 2014-07-01 16:57:28,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 329 (ObjNum 1758 snap 0 seq_num 1758) dirty exists 2014-07-01 16:57:28,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:2792: expect (ObjNum 1299 snap 0 seq_num 1299) 2014-07-01 16:57:28,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1225: done (0 left) 2014-07-01 16:57:28,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: read oid 418 snap -1 2014-07-01 16:57:28,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1227: delete oid 128 current snap is 115 2014-07-01 16:57:28,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1227: done (1 left) 2014-07-01 16:57:28,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1228: snap_remove snap 102 2014-07-01 16:57:28,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: expect (ObjNum 417 snap 0 seq_num 417) 2014-07-01 16:57:29,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: expect (ObjNum 1573 snap 0 seq_num 1573) 2014-07-01 16:57:30,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:1226: done (1 left) 2014-07-01 16:57:30,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1228: done (0 left) 2014-07-01 16:57:30,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1229: rollback oid 55 current snap is 115 2014-07-01 16:57:30,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 55 to 51 2014-07-01 16:57:30,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: read oid 431 snap 79 2014-07-01 16:57:30,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: read oid 349 snap -1 2014-07-01 16:57:30,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: write oid 340 current snap is 115 2014-07-01 16:57:30,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: seq_num 769 ranges {743096=617603,2115644=565232,3203129=422611,3943095=1} 2014-07-01 16:57:30,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0437250-340 from 743096 to 1360699 tid 1 2014-07-01 16:57:30,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0437250-340 from 2115644 to 2680876 tid 2 2014-07-01 16:57:30,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0437250-340 from 3203129 to 3625740 tid 3 2014-07-01 16:57:30,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: writing mira0437250-340 from 3943095 to 3943096 tid 4 2014-07-01 16:57:30,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: delete oid 410 current snap is 115 2014-07-01 16:57:30,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: expect (ObjNum 697 snap 90 seq_num 697) 2014-07-01 16:57:30,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: expect (ObjNum 430 snap 0 seq_num 430) 2014-07-01 16:57:30,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 1 to mira0437250-340 2014-07-01 16:57:30,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 2 to mira0437250-340 2014-07-01 16:57:30,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1230: done (4 left) 2014-07-01 16:57:30,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:1231: done (3 left) 2014-07-01 16:57:30,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1233: done (2 left) 2014-07-01 16:57:30,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: rollback oid 339 current snap is 115 2014-07-01 16:57:30,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 339 to 91 2014-07-01 16:57:30,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:1235: snap_create 2014-07-01 16:57:31,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1235: done (3 left) 2014-07-01 16:57:31,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: write oid 140 current snap is 116 2014-07-01 16:57:31,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: seq_num 771 ranges {749107=713660,1873765=75342} 2014-07-01 16:57:31,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: writing mira0437250-140 from 749107 to 1462767 tid 1 2014-07-01 16:57:31,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: writing mira0437250-140 from 1873765 to 1949107 tid 2 2014-07-01 16:57:31,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: read oid 99 snap -1 2014-07-01 16:57:31,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: snap_create 2014-07-01 16:57:31,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 510 (ObjNum 54 snap 0 seq_num 54) dirty exists 2014-07-01 16:57:32,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 3 to mira0437250-340 2014-07-01 16:57:32,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 4 to mira0437250-340 2014-07-01 16:57:32,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 6 to mira0437250-340 2014-07-01 16:57:32,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: finishing write tid 1 to mira0437250-140 2014-07-01 16:57:32,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: finishing write tid 2 to mira0437250-140 2014-07-01 16:57:32,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 216 (ObjNum 338 snap 0 seq_num 338) dirty exists 2014-07-01 16:57:32,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: finishing write tid 4 to mira0437250-140 2014-07-01 16:57:32,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: expect (ObjNum 1652 snap 0 seq_num 1652) 2014-07-01 16:57:32,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: finishing write tid 5 to mira0437250-140 2014-07-01 16:57:32,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 404 (ObjNum 771 snap 116 seq_num 771) dirty exists 2014-07-01 16:57:32,488.488 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:32,488.488 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:32,488.488 INFO:teuthology.task.thrashosds.thrasher:Killing osd 5, live_osds are [1, 0, 3, 2, 5, 4] 2014-07-01 16:57:32,489.489 DEBUG:teuthology.task.ceph.osd.5:waiting for process to exit 2014-07-01 16:57:32,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1229: done (5 left) 2014-07-01 16:57:32,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1234: done (4 left) 2014-07-01 16:57:32,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1236: done (3 left) 2014-07-01 16:57:32,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1238: done (2 left) 2014-07-01 16:57:32,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:1239: delete oid 293 current snap is 117 2014-07-01 16:57:35,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 1 to mira0435601-200 2014-07-01 16:57:35,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: finishing write tid 1 to mira0435601-492 2014-07-01 16:57:35,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: finishing write tid 2 to mira0435601-492 2014-07-01 16:57:35,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: finishing write tid 4 to mira0435601-492 2014-07-01 16:57:35,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 3 to mira0435601-200 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 4 to mira0435601-200 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 134 (ObjNum 1754 snap 0 seq_num 1754) dirty exists 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: done (14 left) 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: done (13 left) 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: done (12 left) 2014-07-01 16:57:35,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: done (11 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: done (10 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: done (9 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: done (8 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2790: done (7 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: done (6 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2792: done (5 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: done (4 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: done (3 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: done (2 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: done (1 left) 2014-07-01 16:57:35,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: read oid 142 snap -1 2014-07-01 16:57:35,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: finishing write tid 5 to mira0435601-492 2014-07-01 16:57:35,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 185 (ObjNum 1748 snap 0 seq_num 1748) dirty exists 2014-07-01 16:57:35,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: done (1 left) 2014-07-01 16:57:35,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 289 current snap is 0 2014-07-01 16:57:35,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: seq_num 1762 ranges {3416064=729088,4145152=729088,4874240=73728} 2014-07-01 16:57:35,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0435601-289 from 3416064 to 4145152 tid 1 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0435601-289 from 4145152 to 4874240 tid 2 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0435601-289 from 4874240 to 4947968 tid 3 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 442 current snap is 0 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: seq_num 1763 ranges {4132864=651264,4784128=557056,5341184=729088,6070272=319488} 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: writing mira0435601-442 from 4132864 to 4784128 tid 1 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: writing mira0435601-442 from 4784128 to 5341184 tid 2 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: writing mira0435601-442 from 5341184 to 6070272 tid 3 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: writing mira0435601-442 from 6070272 to 6389760 tid 4 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: read oid 371 snap -1 2014-07-01 16:57:35,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 16:57:35,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:2802: read oid 76 snap -1 2014-07-01 16:57:35,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: done (4 left) 2014-07-01 16:57:35,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: done (3 left) 2014-07-01 16:57:35,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 338 current snap is 0 2014-07-01 16:57:35,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: seq_num 1764 ranges {5537792=679936,6217728=401408} 2014-07-01 16:57:35,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: writing mira0435601-338 from 5537792 to 6217728 tid 1 2014-07-01 16:57:35,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: writing mira0435601-338 from 6217728 to 6619136 tid 2 2014-07-01 16:57:35,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 497 current snap is 0 2014-07-01 16:57:35,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: seq_num 1765 ranges {1679360=548864,2228224=401408} 2014-07-01 16:57:35,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: writing mira0435601-497 from 1679360 to 2228224 tid 1 2014-07-01 16:57:35,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: writing mira0435601-497 from 2228224 to 2629632 tid 2 2014-07-01 16:57:35,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: read oid 29 snap -1 2014-07-01 16:57:35,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:2806: read oid 174 snap -1 2014-07-01 16:57:35,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 127 current snap is 0 2014-07-01 16:57:35,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: seq_num 1766 ranges {3944448=421888,4366336=401408} 2014-07-01 16:57:35,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: writing mira0435601-127 from 3944448 to 4366336 tid 1 2014-07-01 16:57:35,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: writing mira0435601-127 from 4366336 to 4767744 tid 2 2014-07-01 16:57:35,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 353 current snap is 0 2014-07-01 16:57:35,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: seq_num 1767 ranges {2547712=446464,2994176=786432,3780608=778240,4558848=40960} 2014-07-01 16:57:35,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: writing mira0435601-353 from 2547712 to 2994176 tid 1 2014-07-01 16:57:35,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: writing mira0435601-353 from 2994176 to 3780608 tid 2 2014-07-01 16:57:35,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: writing mira0435601-353 from 3780608 to 4558848 tid 3 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: writing mira0435601-353 from 4558848 to 4599808 tid 4 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: seq_num 1768 ranges {0=368640} 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: writing mira0435601-444 from 0 to 368640 tid 1 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 16:57:35,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: seq_num 1769 ranges {2781184=241664} 2014-07-01 16:57:35,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: writing mira0435601-208 from 2781184 to 3022848 tid 1 2014-07-01 16:57:36,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 456 current snap is 0 2014-07-01 16:57:36,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: seq_num 1770 ranges {5169152=667648,5836800=536576,6373376=266240} 2014-07-01 16:57:36,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: writing mira0435601-456 from 5169152 to 5836800 tid 1 2014-07-01 16:57:36,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: writing mira0435601-456 from 5836800 to 6373376 tid 2 2014-07-01 16:57:36,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: writing mira0435601-456 from 6373376 to 6639616 tid 3 2014-07-01 16:57:36,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: read oid 475 snap -1 2014-07-01 16:57:36,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: expect (ObjNum 1481 snap 0 seq_num 1481) 2014-07-01 16:57:38,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: finishing write tid 1 to mira0435601-444 2014-07-01 16:57:38,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: finishing write tid 3 to mira0435601-444 2014-07-01 16:57:38,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: finishing write tid 4 to mira0435601-444 2014-07-01 16:57:38,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 220 (ObjNum 1768 snap 0 seq_num 1768) dirty exists 2014-07-01 16:57:38,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: finishing write tid 1 to mira0435601-456 2014-07-01 16:57:38,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: finishing write tid 2 to mira0435601-456 2014-07-01 16:57:38,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: done (12 left) 2014-07-01 16:57:38,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: done (11 left) 2014-07-01 16:57:38,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: read oid 382 snap -1 2014-07-01 16:57:38,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: finishing write tid 3 to mira0435601-456 2014-07-01 16:57:38,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: finishing write tid 5 to mira0435601-456 2014-07-01 16:57:38,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: expect (ObjNum 1222 snap 0 seq_num 1222) 2014-07-01 16:57:38,489.489 INFO:teuthology.task.ceph.osd.5:Stopped 2014-07-01 16:57:39,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: finishing write tid 6 to mira0435601-456 2014-07-01 16:57:39,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 258 (ObjNum 1770 snap 0 seq_num 1770) dirty exists 2014-07-01 16:57:39,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: expect (ObjNum 1391 snap 0 seq_num 1391) 2014-07-01 16:57:40,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: done (11 left) 2014-07-01 16:57:40,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: done (10 left) 2014-07-01 16:57:40,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: done (9 left) 2014-07-01 16:57:40,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 110 current snap is 0 2014-07-01 16:57:40,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: seq_num 1771 ranges {929792=540672,1470464=765952,2236416=36864} 2014-07-01 16:57:40,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: writing mira0435601-110 from 929792 to 1470464 tid 1 2014-07-01 16:57:40,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: writing mira0435601-110 from 1470464 to 2236416 tid 2 2014-07-01 16:57:40,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: writing mira0435601-110 from 2236416 to 2273280 tid 3 2014-07-01 16:57:40,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2815: read oid 117 snap -1 2014-07-01 16:57:40,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: finishing write tid 1 to mira0435601-110 2014-07-01 16:57:40,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 16:57:40,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: seq_num 1772 ranges {7348224=462848,7811072=749568,8560640=495616,9056256=360448} 2014-07-01 16:57:40,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: writing mira0435601-97 from 7348224 to 7811072 tid 1 2014-07-01 16:57:40,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: writing mira0435601-97 from 7811072 to 8560640 tid 2 2014-07-01 16:57:40,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: writing mira0435601-97 from 8560640 to 9056256 tid 3 2014-07-01 16:57:40,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: writing mira0435601-97 from 9056256 to 9416704 tid 4 2014-07-01 16:57:40,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 190 current snap is 0 2014-07-01 16:57:40,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: seq_num 1773 ranges {1818624=487424,2306048=401408} 2014-07-01 16:57:40,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: writing mira0435601-190 from 1818624 to 2306048 tid 1 2014-07-01 16:57:40,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: writing mira0435601-190 from 2306048 to 2707456 tid 2 2014-07-01 16:57:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2818: read oid 350 snap -1 2014-07-01 16:57:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: finishing write tid 2 to mira0435601-110 2014-07-01 16:57:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: finishing write tid 3 to mira0435601-110 2014-07-01 16:57:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: finishing write tid 5 to mira0435601-110 2014-07-01 16:57:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2815: expect (ObjNum 1746 snap 0 seq_num 1746) 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: finishing write tid 6 to mira0435601-110 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 166 (ObjNum 1771 snap 0 seq_num 1771) dirty exists 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: finishing write tid 1 to mira0435601-190 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: finishing write tid 2 to mira0435601-190 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: finishing write tid 4 to mira0435601-190 2014-07-01 16:57:40,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: finishing write tid 5 to mira0435601-190 2014-07-01 16:57:40,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 272 (ObjNum 1773 snap 0 seq_num 1773) dirty exists 2014-07-01 16:57:40,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: done (13 left) 2014-07-01 16:57:40,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2815: done (12 left) 2014-07-01 16:57:40,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: done (11 left) 2014-07-01 16:57:40,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2819: delete oid 297 current snap is 0 2014-07-01 16:57:43,490.490 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2] out_osds: [1] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:57:43,491.491 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:43,492.492 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 16:57:43,493.493 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 16:57:44,922.922 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 16:57:44,949.949 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 16:57:49,949.949 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:57:49,949.949 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:49,949.949 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 16:57:49,950.950 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:57:50,566.566 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:57:53,526.526 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 16:57:54,271.271 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 16:57:56,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2819: done (11 left) 2014-07-01 16:57:56,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2820: read oid 273 snap -1 2014-07-01 16:57:56,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 16:57:56,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: seq_num 1775 ranges {696320=790528,1486848=745472,2232320=458752} 2014-07-01 16:57:56,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: writing mira0435601-228 from 696320 to 1486848 tid 1 2014-07-01 16:57:56,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: writing mira0435601-228 from 1486848 to 2232320 tid 2 2014-07-01 16:57:56,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: writing mira0435601-228 from 2232320 to 2691072 tid 3 2014-07-01 16:57:56,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 200 current snap is 0 2014-07-01 16:57:56,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: seq_num 1776 ranges {6868992=503808} 2014-07-01 16:57:56,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: writing mira0435601-200 from 6868992 to 7372800 tid 1 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: delete oid 23 current snap is 0 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: finishing write tid 1 to mira0435601-127 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: finishing write tid 1 to mira0435601-497 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: finishing write tid 2 to mira0435601-497 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: finishing write tid 4 to mira0435601-497 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: finishing write tid 2 to mira0435601-127 2014-07-01 16:57:56,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: finishing write tid 4 to mira0435601-127 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 1 to mira0435601-353 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: finishing write tid 5 to mira0435601-497 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 350 (ObjNum 1765 snap 0 seq_num 1765) dirty exists 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 1 to mira0435601-97 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 2 to mira0435601-353 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 2 to mira0435601-97 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 3 to mira0435601-353 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 3 to mira0435601-97 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 4 to mira0435601-353 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 6 to mira0435601-353 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 4 to mira0435601-97 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 6 to mira0435601-97 2014-07-01 16:57:56,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: finishing write tid 7 to mira0435601-353 2014-07-01 16:57:56,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 203 (ObjNum 1767 snap 0 seq_num 1767) dirty exists 2014-07-01 16:57:56,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: finishing write tid 5 to mira0435601-127 2014-07-01 16:57:56,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 198 (ObjNum 1766 snap 0 seq_num 1766) dirty exists 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: finishing write tid 7 to mira0435601-97 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 292 (ObjNum 1772 snap 0 seq_num 1772) dirty exists 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: finishing write tid 1 to mira0435601-228 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: finishing write tid 2 to mira0435601-228 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: finishing write tid 3 to mira0435601-228 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: finishing write tid 5 to mira0435601-228 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: finishing write tid 6 to mira0435601-228 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 148 (ObjNum 1775 snap 0 seq_num 1775) dirty exists 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: done (14 left) 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: done (13 left) 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: done (12 left) 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: done (11 left) 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: done (10 left) 2014-07-01 16:57:56,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: done (9 left) 2014-07-01 16:57:56,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 16:57:56,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: seq_num 1778 ranges {5644288=610304,6254592=741376,6995968=61440} 2014-07-01 16:57:56,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: writing mira0435601-394 from 5644288 to 6254592 tid 1 2014-07-01 16:57:56,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: writing mira0435601-394 from 6254592 to 6995968 tid 2 2014-07-01 16:57:56,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: writing mira0435601-394 from 6995968 to 7057408 tid 3 2014-07-01 16:57:56,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 125 current snap is 0 2014-07-01 16:57:56,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: seq_num 1779 ranges {1585152=724992,2310144=569344,2879488=561152,3440640=475136} 2014-07-01 16:57:56,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: writing mira0435601-125 from 1585152 to 2310144 tid 1 2014-07-01 16:57:56,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: writing mira0435601-125 from 2310144 to 2879488 tid 2 2014-07-01 16:57:56,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: writing mira0435601-125 from 2879488 to 3440640 tid 3 2014-07-01 16:57:56,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: writing mira0435601-125 from 3440640 to 3915776 tid 4 2014-07-01 16:57:56,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 309 current snap is 0 2014-07-01 16:57:56,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: seq_num 1780 ranges {1441792=753664,2195456=401408} 2014-07-01 16:57:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: writing mira0435601-309 from 1441792 to 2195456 tid 1 2014-07-01 16:57:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: writing mira0435601-309 from 2195456 to 2596864 tid 2 2014-07-01 16:57:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 276 current snap is 0 2014-07-01 16:57:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: seq_num 1781 ranges {1613824=430080,2043904=401408} 2014-07-01 16:57:56,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: writing mira0435601-276 from 1613824 to 2043904 tid 1 2014-07-01 16:57:56,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: writing mira0435601-276 from 2043904 to 2445312 tid 2 2014-07-01 16:57:56,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:2828: read oid 207 snap -1 2014-07-01 16:57:56,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:2829: delete oid 357 current snap is 0 2014-07-01 16:57:56,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: finishing write tid 1 to mira0435601-394 2014-07-01 16:57:56,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: finishing write tid 2 to mira0435601-394 2014-07-01 16:57:56,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: finishing write tid 3 to mira0435601-394 2014-07-01 16:57:56,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: finishing write tid 5 to mira0435601-394 2014-07-01 16:57:56,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: finishing write tid 6 to mira0435601-394 2014-07-01 16:57:56,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 192 (ObjNum 1778 snap 0 seq_num 1778) dirty exists 2014-07-01 16:57:57,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: finishing write tid 1 to mira0435601-208 2014-07-01 16:57:57,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: finishing write tid 3 to mira0435601-208 2014-07-01 16:57:57,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: done (14 left) 2014-07-01 16:57:57,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2828: done (13 left) 2014-07-01 16:57:57,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2829: done (12 left) 2014-07-01 16:57:57,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 263 current snap is 0 2014-07-01 16:57:57,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: seq_num 1783 ranges {2027520=573440} 2014-07-01 16:57:57,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: writing mira0435601-263 from 2027520 to 2600960 tid 1 2014-07-01 16:57:57,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 16:57:57,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: seq_num 1784 ranges {11235328=602112,11837440=536576,12374016=438272,12812288=229376} 2014-07-01 16:57:57,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: writing mira0435601-439 from 11235328 to 11837440 tid 1 2014-07-01 16:57:57,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: writing mira0435601-439 from 11837440 to 12374016 tid 2 2014-07-01 16:57:57,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: writing mira0435601-439 from 12374016 to 12812288 tid 3 2014-07-01 16:57:57,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: writing mira0435601-439 from 12812288 to 13041664 tid 4 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: read oid 265 snap -1 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: finishing write tid 4 to mira0435601-208 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 353 (ObjNum 1769 snap 0 seq_num 1769) dirty exists 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: finishing write tid 1 to mira0435601-263 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: finishing write tid 3 to mira0435601-263 2014-07-01 16:57:57,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2802: expect (ObjNum 1326 snap 0 seq_num 1326) 2014-07-01 16:57:59,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: finishing write tid 4 to mira0435601-263 2014-07-01 16:57:59,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 257 (ObjNum 1783 snap 0 seq_num 1783) dirty exists 2014-07-01 16:57:59,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2818: expect (ObjNum 1430 snap 0 seq_num 1430) 2014-07-01 16:57:59,300.300 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:57:59,300.300 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:57:59,300.300 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.314285 2014-07-01 16:57:59,300.300 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 0.314284907611' 2014-07-01 16:57:59,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: finishing write tid 7 to mira0437250-340 2014-07-01 16:57:59,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 225 (ObjNum 769 snap 115 seq_num 769) dirty exists 2014-07-01 16:57:59,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: expect (ObjNum 98 snap 0 seq_num 98) 2014-07-01 16:58:00,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2806: expect (ObjNum 1655 snap 0 seq_num 1655) 2014-07-01 16:58:01,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1232: done (2 left) 2014-07-01 16:58:01,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:1237: done (1 left) 2014-07-01 16:58:01,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:1239: done (0 left) 2014-07-01 16:58:01,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:1240: delete oid 352 current snap is 117 2014-07-01 16:58:01,074.074 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 0.314285 (8205962) 2014-07-01 16:58:01,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:1240: done (0 left) 2014-07-01 16:58:01,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:1241: delete oid 201 current snap is 117 2014-07-01 16:58:01,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1241: done (0 left) 2014-07-01 16:58:01,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1242: rollback oid 433 current snap is 117 2014-07-01 16:58:01,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 433 to 79 2014-07-01 16:58:01,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: write oid 332 current snap is 117 2014-07-01 16:58:01,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: seq_num 775 ranges {429226=636973,1626421=771875,2829225=1} 2014-07-01 16:58:01,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: writing mira0437250-332 from 429226 to 1066199 tid 1 2014-07-01 16:58:01,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: writing mira0437250-332 from 1626421 to 2398296 tid 2 2014-07-01 16:58:01,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: writing mira0437250-332 from 2829225 to 2829226 tid 3 2014-07-01 16:58:01,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: snap_create 2014-07-01 16:58:01,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: finishing write tid 1 to mira0437250-332 2014-07-01 16:58:01,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 254 (ObjNum 432 snap 0 seq_num 432) dirty exists 2014-07-01 16:58:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: finishing write tid 2 to mira0437250-332 2014-07-01 16:58:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: finishing write tid 3 to mira0437250-332 2014-07-01 16:58:01,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: finishing write tid 5 to mira0437250-332 2014-07-01 16:58:01,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: finishing write tid 6 to mira0437250-332 2014-07-01 16:58:01,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 417 (ObjNum 775 snap 117 seq_num 775) dirty exists 2014-07-01 16:58:02,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1242: done (2 left) 2014-07-01 16:58:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1243: done (1 left) 2014-07-01 16:58:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1244: done (0 left) 2014-07-01 16:58:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: write oid 16 current snap is 118 2014-07-01 16:58:02,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: seq_num 776 ranges {640808=608777,1820809=442070,2921338=119470} 2014-07-01 16:58:02,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: writing mira0437250-16 from 640808 to 1249585 tid 1 2014-07-01 16:58:02,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: writing mira0437250-16 from 1820809 to 2262879 tid 2 2014-07-01 16:58:02,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: writing mira0437250-16 from 2921338 to 3040808 tid 3 2014-07-01 16:58:02,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: snap_create 2014-07-01 16:58:02,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: finishing write tid 1 to mira0437250-16 2014-07-01 16:58:02,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: finishing write tid 2 to mira0437250-16 2014-07-01 16:58:02,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: finishing write tid 3 to mira0437250-16 2014-07-01 16:58:02,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: finishing write tid 5 to mira0437250-16 2014-07-01 16:58:02,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: finishing write tid 6 to mira0437250-16 2014-07-01 16:58:02,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 252 (ObjNum 776 snap 118 seq_num 776) dirty exists 2014-07-01 16:58:03,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:1245: done (1 left) 2014-07-01 16:58:03,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:1246: done (0 left) 2014-07-01 16:58:03,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: delete oid 16 current snap is 119 2014-07-01 16:58:03,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1247: done (0 left) 2014-07-01 16:58:03,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: read oid 400 snap -1 2014-07-01 16:58:03,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1249: snap_create 2014-07-01 16:58:03,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: expect (ObjNum 399 snap 0 seq_num 399) 2014-07-01 16:58:03,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: expect (ObjNum 846 snap 0 seq_num 846) 2014-07-01 16:58:04,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:1248: done (1 left) 2014-07-01 16:58:04,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1249: done (0 left) 2014-07-01 16:58:04,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: snap_create 2014-07-01 16:58:05,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1250: done (0 left) 2014-07-01 16:58:05,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: write oid 53 current snap is 121 2014-07-01 16:58:05,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: seq_num 778 ranges {529442=1} 2014-07-01 16:58:05,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: writing mira0437250-53 from 529442 to 529443 tid 1 2014-07-01 16:58:05,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: write oid 351 current snap is 121 2014-07-01 16:58:05,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: seq_num 779 ranges {224672=1} 2014-07-01 16:58:05,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: writing mira0437250-351 from 224672 to 224673 tid 1 2014-07-01 16:58:05,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: read oid 90 snap -1 2014-07-01 16:58:05,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:1254: read oid 200 snap -1 2014-07-01 16:58:05,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: read oid 27 snap 91 2014-07-01 16:58:05,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: snap_create 2014-07-01 16:58:05,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: expect (ObjNum 647 snap 67 seq_num 647) 2014-07-01 16:58:05,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: finishing write tid 1 to mira0437250-351 2014-07-01 16:58:05,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:1254: expect (ObjNum 199 snap 0 seq_num 199) 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: finishing write tid 1 to mira0437250-53 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: finishing write tid 3 to mira0437250-53 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: finishing write tid 4 to mira0437250-53 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 386 (ObjNum 778 snap 121 seq_num 778) dirty exists 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: finishing write tid 3 to mira0437250-351 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: finishing write tid 4 to mira0437250-351 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 242 (ObjNum 779 snap 121 seq_num 779) dirty exists 2014-07-01 16:58:05,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: expect (ObjNum 648 snap 69 seq_num 648) 2014-07-01 16:58:05,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 1 to mira0435601-125 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 2 to mira0435601-125 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 3 to mira0435601-125 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 1 to mira0435601-338 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 4 to mira0435601-125 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 2 to mira0435601-338 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 1 to mira0435601-442 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 6 to mira0435601-125 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 4 to mira0435601-338 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 2 to mira0435601-442 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 1 to mira0435601-309 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 3 to mira0435601-442 2014-07-01 16:58:05,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 2 to mira0435601-309 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 4 to mira0435601-442 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: finishing write tid 1 to mira0435601-276 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 6 to mira0435601-442 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 4 to mira0435601-309 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: finishing write tid 2 to mira0435601-276 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 1 to mira0435601-439 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: finishing write tid 4 to mira0435601-276 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 2 to mira0435601-439 2014-07-01 16:58:05,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 3 to mira0435601-439 2014-07-01 16:58:05,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 4 to mira0435601-439 2014-07-01 16:58:05,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 6 to mira0435601-439 2014-07-01 16:58:05,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: finishing write tid 5 to mira0435601-276 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 276 v 210 (ObjNum 1781 snap 0 seq_num 1781) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 7 to mira0435601-125 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 284 (ObjNum 1779 snap 0 seq_num 1779) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 5 to mira0435601-309 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 283 (ObjNum 1780 snap 0 seq_num 1780) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: finishing write tid 7 to mira0435601-442 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 277 (ObjNum 1763 snap 0 seq_num 1763) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 5 to mira0435601-338 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 280 (ObjNum 1764 snap 0 seq_num 1764) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: finishing write tid 7 to mira0435601-439 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 289 (ObjNum 1784 snap 0 seq_num 1784) dirty exists 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 1 to mira0435601-289 2014-07-01 16:58:05,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 2 to mira0435601-289 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 3 to mira0435601-289 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 5 to mira0435601-289 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 6 to mira0435601-289 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 230 (ObjNum 1762 snap 0 seq_num 1762) dirty exists 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: finishing write tid 1 to mira0435601-200 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: finishing write tid 3 to mira0435601-200 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: finishing write tid 4 to mira0435601-200 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 136 (ObjNum 1776 snap 0 seq_num 1776) dirty exists 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: done (14 left) 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: done (13 left) 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2802: done (12 left) 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: done (11 left) 2014-07-01 16:58:05,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2806: done (10 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: done (9 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2818: done (8 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2820: done (7 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: done (6 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: done (5 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: done (4 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: done (3 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: done (2 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: done (1 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: done (0 left) 2014-07-01 16:58:05,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2833: read oid 409 snap -1 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: seq_num 1785 ranges {7323648=315392} 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: writing mira0435601-27 from 7323648 to 7639040 tid 1 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 142 current snap is 0 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: seq_num 1786 ranges {241664=794624,1036288=704512,1740800=786432,2527232=114688} 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: writing mira0435601-142 from 241664 to 1036288 tid 1 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: writing mira0435601-142 from 1036288 to 1740800 tid 2 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: writing mira0435601-142 from 1740800 to 2527232 tid 3 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: writing mira0435601-142 from 2527232 to 2641920 tid 4 2014-07-01 16:58:06,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: read oid 493 snap -1 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: finishing write tid 1 to mira0435601-27 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: finishing write tid 3 to mira0435601-27 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: finishing write tid 4 to mira0435601-27 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 227 (ObjNum 1785 snap 0 seq_num 1785) dirty exists 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: done (3 left) 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: delete oid 250 current snap is 0 2014-07-01 16:58:06,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2833: expect (ObjNum 1628 snap 0 seq_num 1628) 2014-07-01 16:58:06,088.088 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:06,088.088 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:58:06,088.088 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 16:58:06,089.089 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:58:06,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1251: done (5 left) 2014-07-01 16:58:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1252: done (4 left) 2014-07-01 16:58:06,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1253: done (3 left) 2014-07-01 16:58:06,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1254: done (2 left) 2014-07-01 16:58:06,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1255: done (1 left) 2014-07-01 16:58:06,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1256: done (0 left) 2014-07-01 16:58:06,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:1257: snap_remove snap 116 2014-07-01 16:58:06,859.859 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:58:07,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1257: done (0 left) 2014-07-01 16:58:07,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: read oid 492 snap -1 2014-07-01 16:58:07,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1259: snap_create 2014-07-01 16:58:07,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: expect (ObjNum 669 snap 75 seq_num 669) 2014-07-01 16:58:09,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:1258: done (1 left) 2014-07-01 16:58:09,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:1259: done (0 left) 2014-07-01 16:58:09,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: snap_create 2014-07-01 16:58:09,797.797 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 16:58:09,798.798 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 56' 2014-07-01 16:58:10,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:1260: done (0 left) 2014-07-01 16:58:10,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: read oid 60 snap 122 2014-07-01 16:58:10,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: write oid 115 current snap is 124 2014-07-01 16:58:10,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: seq_num 780 ranges {534431=683515,1937370=597061} 2014-07-01 16:58:10,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: writing mira0437250-115 from 534431 to 1217946 tid 1 2014-07-01 16:58:10,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: writing mira0437250-115 from 1937370 to 2534431 tid 2 2014-07-01 16:58:10,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: snap_remove snap 106 2014-07-01 16:58:10,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: expect (ObjNum 59 snap 0 seq_num 59) 2014-07-01 16:58:10,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: finishing write tid 1 to mira0437250-115 2014-07-01 16:58:10,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: finishing write tid 2 to mira0437250-115 2014-07-01 16:58:10,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: finishing write tid 4 to mira0437250-115 2014-07-01 16:58:10,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: finishing write tid 5 to mira0437250-115 2014-07-01 16:58:10,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 229 (ObjNum 780 snap 124 seq_num 780) dirty exists 2014-07-01 16:58:11,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1261: done (2 left) 2014-07-01 16:58:11,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:1262: done (1 left) 2014-07-01 16:58:11,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1263: done (0 left) 2014-07-01 16:58:11,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: write oid 284 current snap is 124 2014-07-01 16:58:11,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: seq_num 781 ranges {630684=406743,1519277=447318,2230683=1} 2014-07-01 16:58:11,213.213 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 56 2014-07-01 16:58:11,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0437250-284 from 630684 to 1037427 tid 1 2014-07-01 16:58:11,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0437250-284 from 1519277 to 1966595 tid 2 2014-07-01 16:58:11,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: writing mira0437250-284 from 2230683 to 2230684 tid 3 2014-07-01 16:58:11,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1265: snap_remove snap 103 2014-07-01 16:58:11,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 1 to mira0437250-284 2014-07-01 16:58:11,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 2 to mira0437250-284 2014-07-01 16:58:11,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 3 to mira0437250-284 2014-07-01 16:58:11,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 5 to mira0437250-284 2014-07-01 16:58:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: finishing write tid 6 to mira0437250-284 2014-07-01 16:58:11,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 536 (ObjNum 781 snap 124 seq_num 781) dirty exists 2014-07-01 16:58:11,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 1 to mira0435601-142 2014-07-01 16:58:11,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 2 to mira0435601-142 2014-07-01 16:58:11,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 3 to mira0435601-142 2014-07-01 16:58:11,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: expect (ObjNum 1049 snap 0 seq_num 1049) 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 4 to mira0435601-142 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 6 to mira0435601-142 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: finishing write tid 7 to mira0435601-142 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 142 v 171 (ObjNum 1786 snap 0 seq_num 1786) dirty exists 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2833: done (3 left) 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: done (2 left) 2014-07-01 16:58:11,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: done (1 left) 2014-07-01 16:58:11,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: done (0 left) 2014-07-01 16:58:11,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 26 current snap is 0 2014-07-01 16:58:11,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: seq_num 1788 ranges {0=692224,692224=401408} 2014-07-01 16:58:11,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: writing mira0435601-26 from 0 to 692224 tid 1 2014-07-01 16:58:11,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: writing mira0435601-26 from 692224 to 1093632 tid 2 2014-07-01 16:58:11,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 16:58:11,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: seq_num 1789 ranges {368640=565248,933888=401408} 2014-07-01 16:58:11,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: writing mira0435601-444 from 368640 to 933888 tid 1 2014-07-01 16:58:11,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: writing mira0435601-444 from 933888 to 1335296 tid 2 2014-07-01 16:58:11,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2840: delete oid 203 current snap is 0 2014-07-01 16:58:11,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 1 to mira0435601-26 2014-07-01 16:58:11,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 2 to mira0435601-26 2014-07-01 16:58:11,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 4 to mira0435601-26 2014-07-01 16:58:11,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 5 to mira0435601-26 2014-07-01 16:58:11,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 213 (ObjNum 1788 snap 0 seq_num 1788) dirty exists 2014-07-01 16:58:11,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: finishing write tid 1 to mira0435601-444 2014-07-01 16:58:11,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: finishing write tid 2 to mira0435601-444 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: finishing write tid 4 to mira0435601-444 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: done (2 left) 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2840: done (1 left) 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: seq_num 1791 ranges {5763072=512000} 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: writing mira0435601-229 from 5763072 to 6275072 tid 1 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2842: delete oid 203 current snap is 0 2014-07-01 16:58:11,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: finishing write tid 5 to mira0435601-444 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 223 (ObjNum 1789 snap 0 seq_num 1789) dirty exists 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: done (2 left) 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2842: done (1 left) 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 106 current snap is 0 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: seq_num 1793 ranges {1613824=409600,2023424=573440,2596864=557056,3153920=73728} 2014-07-01 16:58:11,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0435601-106 from 1613824 to 2023424 tid 1 2014-07-01 16:58:11,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0435601-106 from 2023424 to 2596864 tid 2 2014-07-01 16:58:11,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0435601-106 from 2596864 to 3153920 tid 3 2014-07-01 16:58:11,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0435601-106 from 3153920 to 3227648 tid 4 2014-07-01 16:58:11,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 65 current snap is 0 2014-07-01 16:58:11,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: seq_num 1794 ranges {1396736=684032,2080768=507904,2588672=294912} 2014-07-01 16:58:12,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: writing mira0435601-65 from 1396736 to 2080768 tid 1 2014-07-01 16:58:12,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: writing mira0435601-65 from 2080768 to 2588672 tid 2 2014-07-01 16:58:12,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: writing mira0435601-65 from 2588672 to 2883584 tid 3 2014-07-01 16:58:12,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2845: delete oid 112 current snap is 0 2014-07-01 16:58:12,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2845: done (3 left) 2014-07-01 16:58:12,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 16:58:12,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: seq_num 1796 ranges {6500352=630784,7131136=401408} 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: writing mira0435601-147 from 6500352 to 7131136 tid 1 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: writing mira0435601-147 from 7131136 to 7532544 tid 2 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2847: read oid 267 snap -1 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2848: read oid 445 snap -1 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2849: read oid 215 snap -1 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: delete oid 1 current snap is 0 2014-07-01 16:58:12,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: finishing write tid 1 to mira0435601-147 2014-07-01 16:58:12,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: done (7 left) 2014-07-01 16:58:12,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 341 current snap is 0 2014-07-01 16:58:12,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: seq_num 1798 ranges {2174976=778240} 2014-07-01 16:58:12,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: writing mira0435601-341 from 2174976 to 2953216 tid 1 2014-07-01 16:58:12,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 92 current snap is 0 2014-07-01 16:58:12,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: seq_num 1799 ranges {552960=450560} 2014-07-01 16:58:12,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: writing mira0435601-92 from 552960 to 1003520 tid 1 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: read oid 414 snap -1 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: finishing write tid 2 to mira0435601-147 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: finishing write tid 4 to mira0435601-147 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2854: read oid 62 snap -1 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: finishing write tid 5 to mira0435601-147 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 151 (ObjNum 1796 snap 0 seq_num 1796) dirty exists 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: done (10 left) 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 377 current snap is 0 2014-07-01 16:58:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: seq_num 1800 ranges {4829184=323584} 2014-07-01 16:58:12,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1264: done (1 left) 2014-07-01 16:58:12,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1265: done (0 left) 2014-07-01 16:58:12,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1266: snap_remove snap 93 2014-07-01 16:58:12,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: writing mira0435601-377 from 4829184 to 5152768 tid 1 2014-07-01 16:58:12,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: read oid 449 snap -1 2014-07-01 16:58:12,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2847: expect (ObjNum 1677 snap 0 seq_num 1677) 2014-07-01 16:58:13,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1266: done (0 left) 2014-07-01 16:58:13,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:1267: rollback oid 303 current snap is 124 2014-07-01 16:58:13,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 303 to 115 2014-07-01 16:58:13,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:1268: snap_remove snap 74 2014-07-01 16:58:13,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 539 (ObjNum 302 snap 0 seq_num 302) dirty exists 2014-07-01 16:58:14,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1267: done (1 left) 2014-07-01 16:58:14,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1268: done (0 left) 2014-07-01 16:58:14,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: read oid 243 snap -1 2014-07-01 16:58:14,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: delete oid 402 current snap is 124 2014-07-01 16:58:14,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: expect (ObjNum 585 snap 35 seq_num 585) 2014-07-01 16:58:14,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:1269: done (1 left) 2014-07-01 16:58:14,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:1270: done (0 left) 2014-07-01 16:58:14,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:1271: delete oid 443 current snap is 124 2014-07-01 16:58:14,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1271: done (0 left) 2014-07-01 16:58:14,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:1272: snap_create 2014-07-01 16:58:15,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1272: done (0 left) 2014-07-01 16:58:15,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1273: snap_remove snap 98 2014-07-01 16:58:15,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2849: expect (ObjNum 1687 snap 0 seq_num 1687) 2014-07-01 16:58:16,226.226 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:16,227.227 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:58:16,227.227 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.367165 2014-07-01 16:58:16,228.228 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 0.367165161426' 2014-07-01 16:58:16,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1273: done (0 left) 2014-07-01 16:58:16,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: write oid 156 current snap is 125 2014-07-01 16:58:16,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: seq_num 784 ranges {613526=606828,1805354=692976,3043142=370384} 2014-07-01 16:58:16,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: writing mira0437250-156 from 613526 to 1220354 tid 1 2014-07-01 16:58:16,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: writing mira0437250-156 from 1805354 to 2498330 tid 2 2014-07-01 16:58:16,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: writing mira0437250-156 from 3043142 to 3413526 tid 3 2014-07-01 16:58:16,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1275: delete oid 21 current snap is 125 2014-07-01 16:58:16,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 1 to mira0437250-156 2014-07-01 16:58:16,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 2 to mira0437250-156 2014-07-01 16:58:16,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 3 to mira0437250-156 2014-07-01 16:58:16,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 5 to mira0437250-156 2014-07-01 16:58:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1275: done (1 left) 2014-07-01 16:58:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: write oid 409 current snap is 125 2014-07-01 16:58:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: seq_num 786 ranges {370960=1} 2014-07-01 16:58:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: writing mira0437250-409 from 370960 to 370961 tid 1 2014-07-01 16:58:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1277: rollback oid 385 current snap is 125 2014-07-01 16:58:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 385 to 51 2014-07-01 16:58:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: finishing write tid 6 to mira0437250-156 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 370 (ObjNum 784 snap 125 seq_num 784) dirty exists 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1274: done (2 left) 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1278: rollback oid 321 current snap is 125 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 321 to 91 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: snap_remove snap 110 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 385 (ObjNum 630 snap 57 seq_num 630) dirty exists 2014-07-01 16:58:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 1 to mira0437250-409 2014-07-01 16:58:16,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 3 to mira0437250-409 2014-07-01 16:58:16,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: finishing write tid 4 to mira0437250-409 2014-07-01 16:58:16,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 232 (ObjNum 786 snap 125 seq_num 786) dirty exists 2014-07-01 16:58:16,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 385 v 442 (ObjNum 384 snap 0 seq_num 384) dirty exists 2014-07-01 16:58:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1276: done (3 left) 2014-07-01 16:58:17,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1277: done (2 left) 2014-07-01 16:58:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1278: done (1 left) 2014-07-01 16:58:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1279: done (0 left) 2014-07-01 16:58:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1280: read oid 255 snap -1 2014-07-01 16:58:17,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: read oid 386 snap -1 2014-07-01 16:58:17,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: rollback oid 232 current snap is 125 2014-07-01 16:58:17,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 232 to 71 2014-07-01 16:58:17,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: snap_create 2014-07-01 16:58:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1280: expect (ObjNum 254 snap 0 seq_num 254) 2014-07-01 16:58:17,280.280 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 0.367165 (8240622) 2014-07-01 16:58:17,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 633 (ObjNum 231 snap 0 seq_num 231) dirty exists 2014-07-01 16:58:18,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1280: done (3 left) 2014-07-01 16:58:18,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1281: done (2 left) 2014-07-01 16:58:18,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1282: done (1 left) 2014-07-01 16:58:18,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1283: done (0 left) 2014-07-01 16:58:18,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: read oid 288 snap -1 2014-07-01 16:58:18,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: write oid 181 current snap is 126 2014-07-01 16:58:18,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: seq_num 787 ranges {714767=400000} 2014-07-01 16:58:18,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: writing mira0437250-181 from 714767 to 1114767 tid 1 2014-07-01 16:58:18,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:1286: rollback oid 42 current snap is 126 2014-07-01 16:58:18,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 42 to 51 2014-07-01 16:58:18,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: read oid 115 snap -1 2014-07-01 16:58:18,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:1288: delete oid 381 current snap is 126 2014-07-01 16:58:18,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: expect (ObjNum 755 snap 110 seq_num 755) 2014-07-01 16:58:18,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: finishing write tid 1 to mira0437250-181 2014-07-01 16:58:18,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: finishing write tid 3 to mira0437250-181 2014-07-01 16:58:18,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: finishing write tid 4 to mira0437250-181 2014-07-01 16:58:18,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 414 (ObjNum 787 snap 126 seq_num 787) dirty exists 2014-07-01 16:58:18,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: expect (ObjNum 780 snap 124 seq_num 780) 2014-07-01 16:58:18,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2854: expect (ObjNum 61 snap 0 seq_num 61) 2014-07-01 16:58:19,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 446 (ObjNum 516 snap 9 seq_num 516) dirty exists 2014-07-01 16:58:19,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1284: done (4 left) 2014-07-01 16:58:19,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1285: done (3 left) 2014-07-01 16:58:19,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1286: done (2 left) 2014-07-01 16:58:19,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1287: done (1 left) 2014-07-01 16:58:19,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:1288: done (0 left) 2014-07-01 16:58:19,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:1289: rollback oid 471 current snap is 126 2014-07-01 16:58:19,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 471 to 85 2014-07-01 16:58:19,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:1290: snap_create 2014-07-01 16:58:19,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 345 (ObjNum 470 snap 0 seq_num 470) dirty exists 2014-07-01 16:58:20,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1289: done (1 left) 2014-07-01 16:58:20,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1290: done (0 left) 2014-07-01 16:58:20,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1291: read oid 85 snap -1 2014-07-01 16:58:20,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1292: read oid 434 snap 53 2014-07-01 16:58:20,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:1293: snap_remove snap 124 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 1 to mira0435601-229 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 3 to mira0435601-229 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: finishing write tid 1 to mira0435601-92 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: finishing write tid 3 to mira0435601-92 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 4 to mira0435601-229 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 256 (ObjNum 1791 snap 0 seq_num 1791) dirty exists 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: finishing write tid 4 to mira0435601-92 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 258 (ObjNum 1799 snap 0 seq_num 1799) dirty exists 2014-07-01 16:58:20,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: expect (ObjNum 1590 snap 0 seq_num 1590) 2014-07-01 16:58:20,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1291: expect (ObjNum 84 snap 0 seq_num 84) 2014-07-01 16:58:20,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1292: expect (ObjNum 556 snap 25 seq_num 556) 2014-07-01 16:58:21,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 1 to mira0435601-377 2014-07-01 16:58:21,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 1 to mira0435601-106 2014-07-01 16:58:21,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 3 to mira0435601-377 2014-07-01 16:58:21,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: finishing write tid 1 to mira0435601-65 2014-07-01 16:58:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: finishing write tid 2 to mira0435601-65 2014-07-01 16:58:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 4 to mira0435601-377 2014-07-01 16:58:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 259 (ObjNum 1800 snap 0 seq_num 1800) dirty exists 2014-07-01 16:58:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2848: expect (ObjNum 1582 snap 0 seq_num 1582) 2014-07-01 16:58:22,298.298 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 2, 1] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:22,298.298 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:58:22,299.299 INFO:teuthology.task.thrashosds.thrasher:Removing osd 2, in_osds are: [3, 0, 5, 4, 2, 1] 2014-07-01 16:58:22,299.299 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 2' 2014-07-01 16:58:22,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:1291: done (2 left) 2014-07-01 16:58:22,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:1292: done (1 left) 2014-07-01 16:58:22,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:1293: done (0 left) 2014-07-01 16:58:22,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: read oid 388 snap 80 2014-07-01 16:58:22,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: write oid 238 current snap is 127 2014-07-01 16:58:22,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: seq_num 789 ranges {471178=405818,1469775=601403} 2014-07-01 16:58:22,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: writing mira0437250-238 from 471178 to 876996 tid 1 2014-07-01 16:58:22,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: writing mira0437250-238 from 1469775 to 2071178 tid 2 2014-07-01 16:58:22,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: write oid 12 current snap is 127 2014-07-01 16:58:22,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: seq_num 790 ranges {416008=661270,1716131=465571,2695422=120586} 2014-07-01 16:58:22,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: writing mira0437250-12 from 416008 to 1077278 tid 1 2014-07-01 16:58:22,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: writing mira0437250-12 from 1716131 to 2181702 tid 2 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: writing mira0437250-12 from 2695422 to 2816008 tid 3 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: write oid 293 current snap is 127 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: seq_num 791 ranges {512262=785522,1699063=687190,2512261=1} 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: writing mira0437250-293 from 512262 to 1297784 tid 1 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: writing mira0437250-293 from 1699063 to 2386253 tid 2 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: writing mira0437250-293 from 2512261 to 2512262 tid 3 2014-07-01 16:58:22,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: write oid 301 current snap is 127 2014-07-01 16:58:22,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: seq_num 792 ranges {455043=758351,1909323=507785,2964624=290419} 2014-07-01 16:58:22,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: writing mira0437250-301 from 455043 to 1213394 tid 1 2014-07-01 16:58:22,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: writing mira0437250-301 from 1909323 to 2417108 tid 2 2014-07-01 16:58:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: writing mira0437250-301 from 2964624 to 3255043 tid 3 2014-07-01 16:58:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1299: snap_remove snap 119 2014-07-01 16:58:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: expect (ObjNum 650 snap 70 seq_num 650) 2014-07-01 16:58:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 1 to mira0437250-238 2014-07-01 16:58:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 2 to mira0437250-238 2014-07-01 16:58:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 4 to mira0437250-238 2014-07-01 16:58:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: finishing write tid 5 to mira0437250-238 2014-07-01 16:58:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 390 (ObjNum 789 snap 127 seq_num 789) dirty exists 2014-07-01 16:58:23,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: finishing write tid 1 to mira0437250-12 2014-07-01 16:58:23,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: finishing write tid 2 to mira0437250-12 2014-07-01 16:58:23,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: finishing write tid 3 to mira0437250-12 2014-07-01 16:58:23,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: finishing write tid 5 to mira0437250-12 2014-07-01 16:58:23,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: finishing write tid 6 to mira0437250-12 2014-07-01 16:58:23,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 237 (ObjNum 790 snap 127 seq_num 790) dirty exists 2014-07-01 16:58:23,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: finishing write tid 1 to mira0437250-293 2014-07-01 16:58:23,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: finishing write tid 2 to mira0437250-293 2014-07-01 16:58:23,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: finishing write tid 3 to mira0437250-293 2014-07-01 16:58:23,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: finishing write tid 5 to mira0437250-293 2014-07-01 16:58:23,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: finishing write tid 6 to mira0437250-293 2014-07-01 16:58:23,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 350 (ObjNum 791 snap 127 seq_num 791) dirty exists 2014-07-01 16:58:23,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: finishing write tid 1 to mira0437250-301 2014-07-01 16:58:23,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: finishing write tid 2 to mira0437250-301 2014-07-01 16:58:23,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: finishing write tid 3 to mira0437250-301 2014-07-01 16:58:23,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: finishing write tid 5 to mira0437250-301 2014-07-01 16:58:23,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: finishing write tid 6 to mira0437250-301 2014-07-01 16:58:23,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 452 (ObjNum 792 snap 127 seq_num 792) dirty exists 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1294: done (5 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1295: done (4 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1296: done (3 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1297: done (2 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1298: done (1 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1299: done (0 left) 2014-07-01 16:58:23,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:1300: delete oid 180 current snap is 127 2014-07-01 16:58:23,324.324 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.2. 2014-07-01 16:58:23,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1300: done (0 left) 2014-07-01 16:58:23,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: rollback oid 212 current snap is 127 2014-07-01 16:58:23,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 212 to 114 2014-07-01 16:58:23,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: write oid 68 current snap is 127 2014-07-01 16:58:23,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: seq_num 794 ranges {212478=1} 2014-07-01 16:58:23,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: writing mira0437250-68 from 212478 to 212479 tid 1 2014-07-01 16:58:23,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: read oid 363 snap -1 2014-07-01 16:58:23,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1304: delete oid 421 current snap is 127 2014-07-01 16:58:23,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1304: done (3 left) 2014-07-01 16:58:23,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: read oid 268 snap 109 2014-07-01 16:58:23,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: read oid 37 snap -1 2014-07-01 16:58:23,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: rollback oid 283 current snap is 127 2014-07-01 16:58:23,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 283 to 87 2014-07-01 16:58:23,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: rollback oid 267 current snap is 127 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 267 to 85 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: write oid 223 current snap is 127 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: seq_num 796 ranges {501114=681664,1701113=1} 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: writing mira0437250-223 from 501114 to 1182778 tid 1 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: writing mira0437250-223 from 1701113 to 1701114 tid 2 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1310: delete oid 297 current snap is 127 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 515 (ObjNum 685 snap 83 seq_num 685) dirty exists 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 392 (ObjNum 266 snap 0 seq_num 266) dirty exists 2014-07-01 16:58:23,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: expect (ObjNum 668 snap 75 seq_num 668) 2014-07-01 16:58:24,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 414 (ObjNum 211 snap 0 seq_num 211) dirty exists 2014-07-01 16:58:24,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: expect (ObjNum 571 snap 27 seq_num 571) 2014-07-01 16:58:24,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: finishing write tid 3 to mira0435601-65 2014-07-01 16:58:24,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: finishing write tid 5 to mira0435601-65 2014-07-01 16:58:24,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: finishing write tid 6 to mira0435601-65 2014-07-01 16:58:24,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 155 (ObjNum 1794 snap 0 seq_num 1794) dirty exists 2014-07-01 16:58:24,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 2 to mira0435601-106 2014-07-01 16:58:24,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: done (11 left) 2014-07-01 16:58:24,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: done (10 left) 2014-07-01 16:58:24,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2847: done (9 left) 2014-07-01 16:58:24,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2848: done (8 left) 2014-07-01 16:58:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2849: done (7 left) 2014-07-01 16:58:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: done (6 left) 2014-07-01 16:58:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: done (5 left) 2014-07-01 16:58:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2854: done (4 left) 2014-07-01 16:58:24,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: done (3 left) 2014-07-01 16:58:24,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 254 current snap is 0 2014-07-01 16:58:24,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: seq_num 1801 ranges {2088960=753664,2842624=778240,3620864=24576} 2014-07-01 16:58:24,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: writing mira0435601-254 from 2088960 to 2842624 tid 1 2014-07-01 16:58:24,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: writing mira0435601-254 from 2842624 to 3620864 tid 2 2014-07-01 16:58:24,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: writing mira0435601-254 from 3620864 to 3645440 tid 3 2014-07-01 16:58:24,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 71 current snap is 0 2014-07-01 16:58:24,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: seq_num 1802 ranges {3497984=622592,4120576=765952,4886528=36864} 2014-07-01 16:58:24,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: writing mira0435601-71 from 3497984 to 4120576 tid 1 2014-07-01 16:58:24,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: writing mira0435601-71 from 4120576 to 4886528 tid 2 2014-07-01 16:58:24,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: writing mira0435601-71 from 4886528 to 4923392 tid 3 2014-07-01 16:58:24,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 35 current snap is 0 2014-07-01 16:58:24,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: seq_num 1803 ranges {4120576=495616,4616192=544768,5160960=258048} 2014-07-01 16:58:24,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: writing mira0435601-35 from 4120576 to 4616192 tid 1 2014-07-01 16:58:24,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: writing mira0435601-35 from 4616192 to 5160960 tid 2 2014-07-01 16:58:24,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: writing mira0435601-35 from 5160960 to 5419008 tid 3 2014-07-01 16:58:24,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 140 current snap is 0 2014-07-01 16:58:24,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: seq_num 1804 ranges {7540736=573440,8114176=401408} 2014-07-01 16:58:24,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: writing mira0435601-140 from 7540736 to 8114176 tid 1 2014-07-01 16:58:24,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: writing mira0435601-140 from 8114176 to 8515584 tid 2 2014-07-01 16:58:24,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: read oid 289 snap -1 2014-07-01 16:58:24,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 3 to mira0435601-106 2014-07-01 16:58:24,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: expect (ObjNum 1231 snap 0 seq_num 1231) 2014-07-01 16:58:25,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 4 to mira0435601-106 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 6 to mira0435601-106 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: finishing write tid 1 to mira0435601-341 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: finishing write tid 3 to mira0435601-341 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 7 to mira0435601-106 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 255 (ObjNum 1793 snap 0 seq_num 1793) dirty exists 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: finishing write tid 4 to mira0435601-341 2014-07-01 16:58:25,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 341 v 257 (ObjNum 1798 snap 0 seq_num 1798) dirty exists 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: finishing write tid 1 to mira0435601-254 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: done (7 left) 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: done (6 left) 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: done (5 left) 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: read oid 302 snap -1 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: finishing write tid 2 to mira0435601-254 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: finishing write tid 3 to mira0435601-254 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2863: read oid 460 snap -1 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: finishing write tid 5 to mira0435601-254 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: finishing write tid 6 to mira0435601-254 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 208 (ObjNum 1801 snap 0 seq_num 1801) dirty exists 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: finishing write tid 1 to mira0435601-35 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: finishing write tid 2 to mira0435601-35 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 1 to mira0435601-140 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: done (6 left) 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: read oid 144 snap -1 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 2 to mira0435601-140 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 4 to mira0435601-140 2014-07-01 16:58:25,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: finishing write tid 3 to mira0435601-35 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: finishing write tid 5 to mira0435601-35 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 5 to mira0435601-140 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 155 (ObjNum 1804 snap 0 seq_num 1804) dirty exists 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: finishing write tid 6 to mira0435601-35 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 296 (ObjNum 1803 snap 0 seq_num 1803) dirty exists 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: finishing write tid 1 to mira0435601-71 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: finishing write tid 2 to mira0435601-71 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: finishing write tid 3 to mira0435601-71 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: finishing write tid 5 to mira0435601-71 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: finishing write tid 6 to mira0435601-71 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 227 (ObjNum 1802 snap 0 seq_num 1802) dirty exists 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: done (6 left) 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: done (5 left) 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: done (4 left) 2014-07-01 16:58:25,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: read oid 241 snap -1 2014-07-01 16:58:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2866: read oid 140 snap -1 2014-07-01 16:58:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: delete oid 91 current snap is 0 2014-07-01 16:58:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: expect (ObjNum 1310 snap 0 seq_num 1310) 2014-07-01 16:58:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2863: expect (ObjNum 1713 snap 0 seq_num 1713) 2014-07-01 16:58:25,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: expect (ObjNum 362 snap 0 seq_num 362) 2014-07-01 16:58:25,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1301: done (8 left) 2014-07-01 16:58:25,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:1303: done (7 left) 2014-07-01 16:58:25,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:1305: done (6 left) 2014-07-01 16:58:25,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:1306: done (5 left) 2014-07-01 16:58:25,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:1307: done (4 left) 2014-07-01 16:58:25,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:1308: done (3 left) 2014-07-01 16:58:25,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:1310: done (2 left) 2014-07-01 16:58:25,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: snap_create 2014-07-01 16:58:26,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 1 to mira0437250-223 2014-07-01 16:58:26,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 2 to mira0437250-223 2014-07-01 16:58:26,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 4 to mira0437250-223 2014-07-01 16:58:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: finishing write tid 5 to mira0437250-223 2014-07-01 16:58:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 312 (ObjNum 796 snap 127 seq_num 796) dirty exists 2014-07-01 16:58:26,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:1309: done (2 left) 2014-07-01 16:58:26,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1311: done (1 left) 2014-07-01 16:58:26,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1312: read oid 30 snap -1 2014-07-01 16:58:26,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1313: rollback oid 27 current snap is 128 2014-07-01 16:58:26,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 87 2014-07-01 16:58:26,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: read oid 153 snap 71 2014-07-01 16:58:26,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: snap_create 2014-07-01 16:58:26,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: expect (ObjNum 152 snap 0 seq_num 152) 2014-07-01 16:58:28,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: expect (ObjNum 1657 snap 0 seq_num 1657) 2014-07-01 16:58:28,341.341 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 1] out_osds: [2] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:28,342.342 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:58:28,342.342 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 16:58:28,342.342 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 16:58:28,343.343 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 16:58:28,343.343 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 16:58:28,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1314: done (4 left) 2014-07-01 16:58:28,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1315: done (3 left) 2014-07-01 16:58:28,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: write oid 82 current snap is 129 2014-07-01 16:58:28,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: seq_num 798 ranges {657526=660345,1819076=564028,2657525=1} 2014-07-01 16:58:28,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: writing mira0437250-82 from 657526 to 1317871 tid 1 2014-07-01 16:58:28,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: writing mira0437250-82 from 1819076 to 2383104 tid 2 2014-07-01 16:58:28,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: writing mira0437250-82 from 2657525 to 2657526 tid 3 2014-07-01 16:58:28,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: rollback oid 477 current snap is 129 2014-07-01 16:58:28,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 477 to 80 2014-07-01 16:58:28,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:1318: snap_remove snap 123 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: finishing write tid 1 to mira0437250-82 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: finishing write tid 2 to mira0437250-82 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: finishing write tid 3 to mira0437250-82 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: finishing write tid 5 to mira0437250-82 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: finishing write tid 6 to mira0437250-82 2014-07-01 16:58:28,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 540 (ObjNum 798 snap 129 seq_num 798) dirty exists 2014-07-01 16:58:28,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 639 (ObjNum 680 snap 79 seq_num 680) dirty exists 2014-07-01 16:58:28,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:1312: expect (ObjNum 29 snap 0 seq_num 29) 2014-07-01 16:58:29,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:1312: done (5 left) 2014-07-01 16:58:29,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1316: done (4 left) 2014-07-01 16:58:29,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1317: done (3 left) 2014-07-01 16:58:29,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:1318: done (2 left) 2014-07-01 16:58:29,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: read oid 170 snap -1 2014-07-01 16:58:29,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1320: read oid 444 snap -1 2014-07-01 16:58:29,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: snap_remove snap 100 2014-07-01 16:58:29,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:1320: expect (ObjNum 727 snap 98 seq_num 727) 2014-07-01 16:58:31,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: expect (ObjNum 559 snap 25 seq_num 559) 2014-07-01 16:58:31,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 352 (ObjNum 647 snap 67 seq_num 647) dirty exists 2014-07-01 16:58:31,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 1 to mira0437250-68 2014-07-01 16:58:31,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 3 to mira0437250-68 2014-07-01 16:58:31,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: finishing write tid 4 to mira0437250-68 2014-07-01 16:58:31,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 68 v 355 (ObjNum 794 snap 127 seq_num 794) dirty exists 2014-07-01 16:58:31,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1302: done (4 left) 2014-07-01 16:58:31,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1313: done (3 left) 2014-07-01 16:58:31,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1319: done (2 left) 2014-07-01 16:58:31,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1320: done (1 left) 2014-07-01 16:58:31,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:1321: done (0 left) 2014-07-01 16:58:31,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:1322: delete oid 72 current snap is 129 2014-07-01 16:58:31,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:1322: done (0 left) 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: write oid 452 current snap is 129 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: seq_num 800 ranges {146160=1} 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: writing mira0437250-452 from 146160 to 146161 tid 1 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: read oid 54 snap -1 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: write oid 350 current snap is 129 2014-07-01 16:58:31,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: seq_num 801 ranges {641391=479630,1840704=525014,2816485=713124,3841390=1} 2014-07-01 16:58:31,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: writing mira0437250-350 from 641391 to 1121021 tid 1 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: writing mira0437250-350 from 1840704 to 2365718 tid 2 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: writing mira0437250-350 from 2816485 to 3529609 tid 3 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: writing mira0437250-350 from 3841390 to 3841391 tid 4 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: write oid 348 current snap is 129 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: seq_num 802 ranges {736621=735082,2237460=630281,3536620=1} 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0437250-348 from 736621 to 1471703 tid 1 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0437250-348 from 2237460 to 2867741 tid 2 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: writing mira0437250-348 from 3536620 to 3536621 tid 3 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: write oid 489 current snap is 129 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: seq_num 803 ranges {646379=459335,1517545=328834} 2014-07-01 16:58:31,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: writing mira0437250-489 from 646379 to 1105714 tid 1 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: writing mira0437250-489 from 1517545 to 1846379 tid 2 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1328: snap_create 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 1 to mira0437250-452 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 3 to mira0437250-452 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: finishing write tid 4 to mira0437250-452 2014-07-01 16:58:31,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 417 (ObjNum 800 snap 129 seq_num 800) dirty exists 2014-07-01 16:58:31,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 1 to mira0437250-350 2014-07-01 16:58:31,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 2 to mira0437250-350 2014-07-01 16:58:31,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 3 to mira0437250-350 2014-07-01 16:58:31,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 4 to mira0437250-350 2014-07-01 16:58:31,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 6 to mira0437250-350 2014-07-01 16:58:32,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:1323: done (5 left) 2014-07-01 16:58:32,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1328: done (4 left) 2014-07-01 16:58:32,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:1329: rollback oid 78 current snap is 130 2014-07-01 16:58:32,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 78 to 125 2014-07-01 16:58:32,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: snap_remove snap 109 2014-07-01 16:58:32,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 0 (ObjNum 27 snap 0 seq_num 0) dirty dne 2014-07-01 16:58:32,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: finishing write tid 7 to mira0437250-350 2014-07-01 16:58:32,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 443 (ObjNum 801 snap 129 seq_num 801) dirty exists 2014-07-01 16:58:33,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 1 to mira0437250-348 2014-07-01 16:58:33,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 2 to mira0437250-348 2014-07-01 16:58:33,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 3 to mira0437250-348 2014-07-01 16:58:33,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 5 to mira0437250-348 2014-07-01 16:58:33,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: finishing write tid 6 to mira0437250-348 2014-07-01 16:58:33,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 656 (ObjNum 802 snap 129 seq_num 802) dirty exists 2014-07-01 16:58:33,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 1 to mira0437250-489 2014-07-01 16:58:33,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 2 to mira0437250-489 2014-07-01 16:58:33,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 4 to mira0437250-489 2014-07-01 16:58:33,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: finishing write tid 5 to mira0437250-489 2014-07-01 16:58:33,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 660 (ObjNum 803 snap 129 seq_num 803) dirty exists 2014-07-01 16:58:33,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: expect (ObjNum 53 snap 0 seq_num 53) 2014-07-01 16:58:33,767.767 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 1] out_osds: [2] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:33,767.767 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 16:58:33,768.768 INFO:teuthology.task.thrashosds.thrasher:Adding osd 2 2014-07-01 16:58:33,768.768 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 2' 2014-07-01 16:58:34,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1324: done (5 left) 2014-07-01 16:58:34,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1325: done (4 left) 2014-07-01 16:58:34,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1326: done (3 left) 2014-07-01 16:58:34,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1327: done (2 left) 2014-07-01 16:58:34,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1329: done (1 left) 2014-07-01 16:58:34,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1330: done (0 left) 2014-07-01 16:58:34,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: delete oid 392 current snap is 130 2014-07-01 16:58:34,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:2866: expect (ObjNum 1804 snap 0 seq_num 1804) 2014-07-01 16:58:34,505.505 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.2. 2014-07-01 16:58:34,513.513 INFO:teuthology.task.thrashosds.thrasher:Added osd 2 2014-07-01 16:58:39,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: expect (ObjNum 986 snap 0 seq_num 986) 2014-07-01 16:58:39,511.511 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 1, 2] out_osds: [] dead_osds: [5] live_osds: [1, 0, 3, 2, 4] 2014-07-01 16:58:39,511.511 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 5 2014-07-01 16:58:39,511.511 INFO:teuthology.task.ceph.osd.5:Restarting daemon 2014-07-01 16:58:39,511.511 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 5' 2014-07-01 16:58:39,844.844 INFO:teuthology.task.ceph.osd.5:Started 2014-07-01 16:58:39,845.845 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok dump_ops_in_flight' 2014-07-01 16:58:40,002.002 INFO:teuthology.task.ceph.osd.5.mira028.stdout:starting osd.5 at :/0 osd_data /var/lib/ceph/osd/ceph-5 /var/lib/ceph/osd/ceph-5/journal 2014-07-01 16:58:40,020.020 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.928529 7f3830529780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 16:58:40,078.078 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987358 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f3cef016/mira0435601-109/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,078.078 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987425 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b511d236/mira0435601-29/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,079.079 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987481 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 6305a436/mira0435601-445/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,080.080 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987542 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find fe937456/mira0435601-320/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,081.081 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987599 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 853c5f96/mira0435601-289/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,081.081 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987655 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 140de3b6/mira0435601-45/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,081.081 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987733 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 76b41ab6/mira0435601-486/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,082.082 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987794 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 46696d6/mira0435601-42/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,179.179 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987889 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ac1bccd6/mira0435601-268/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,179.179 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.987996 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9af000f6/mira0435601-3/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,179.179 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.988088 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 357e45f6/mira0435601-340/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,179.179 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 07:58:39.988134 7f3830529780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 136b58f6/mira0435601-206/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:no valid command found; 10 closest matches: 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:config show 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:help 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:log dump 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:get_command_descriptions 2014-07-01 16:58:40,250.250 INFO:teuthology.orchestra.run.mira028.stderr:git_version 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:config set [...] 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:version 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:2 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:config get 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:0 2014-07-01 16:58:40,251.251 INFO:teuthology.orchestra.run.mira028.stderr:admin_socket: invalid command 2014-07-01 16:58:40,255.255 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 5, ['dump_ops_in_flight'] 2014-07-01 16:58:43,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: expect (ObjNum 1762 snap 0 seq_num 1762) 2014-07-01 16:58:44,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1331: done (0 left) 2014-07-01 16:58:44,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1332: rollback oid 64 current snap is 130 2014-07-01 16:58:44,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 64 to 104 2014-07-01 16:58:44,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: write oid 345 current snap is 130 2014-07-01 16:58:44,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: seq_num 805 ranges {437862=402943,1237861=1} 2014-07-01 16:58:44,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: writing mira0437250-345 from 437862 to 840805 tid 1 2014-07-01 16:58:44,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: writing mira0437250-345 from 1237861 to 1237862 tid 2 2014-07-01 16:58:44,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: write oid 75 current snap is 130 2014-07-01 16:58:44,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: seq_num 806 ranges {631268=659420,1922022=511062,2987285=43983} 2014-07-01 16:58:44,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: writing mira0437250-75 from 631268 to 1290688 tid 1 2014-07-01 16:58:44,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: writing mira0437250-75 from 1922022 to 2433084 tid 2 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: writing mira0437250-75 from 2987285 to 3031268 tid 3 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: write oid 309 current snap is 130 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: seq_num 807 ranges {442850=782648,1751482=732681,3055262=187588} 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: writing mira0437250-309 from 442850 to 1225498 tid 1 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: writing mira0437250-309 from 1751482 to 2484163 tid 2 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: writing mira0437250-309 from 3055262 to 3242850 tid 3 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: snap_remove snap 64 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 309 (ObjNum 63 snap 0 seq_num 63) dirty exists 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: finishing write tid 1 to mira0437250-345 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: finishing write tid 2 to mira0437250-345 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: finishing write tid 4 to mira0437250-345 2014-07-01 16:58:45,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: finishing write tid 5 to mira0437250-345 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 345 v 451 (ObjNum 805 snap 130 seq_num 805) dirty exists 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 1 to mira0437250-309 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 2 to mira0437250-309 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 3 to mira0437250-309 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 5 to mira0437250-309 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: finishing write tid 6 to mira0437250-309 2014-07-01 16:58:45,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 328 (ObjNum 807 snap 130 seq_num 807) dirty exists 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1332: done (4 left) 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1333: done (3 left) 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1335: done (2 left) 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1336: done (1 left) 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: write oid 456 current snap is 130 2014-07-01 16:58:45,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: seq_num 808 ranges {539103=754452,1865613=554300,2891558=47545} 2014-07-01 16:58:45,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: writing mira0437250-456 from 539103 to 1293555 tid 1 2014-07-01 16:58:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: writing mira0437250-456 from 1865613 to 2419913 tid 2 2014-07-01 16:58:45,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: writing mira0437250-456 from 2891558 to 2939103 tid 3 2014-07-01 16:58:45,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1338: read oid 255 snap 104 2014-07-01 16:58:45,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: write oid 410 current snap is 130 2014-07-01 16:58:45,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: seq_num 809 ranges {731484=1} 2014-07-01 16:58:45,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: writing mira0437250-410 from 731484 to 731485 tid 1 2014-07-01 16:58:45,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: delete oid 376 current snap is 130 2014-07-01 16:58:45,257.257 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok dump_ops_in_flight' 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: finishing write tid 1 to mira0437250-456 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: finishing write tid 2 to mira0437250-456 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1340: done (4 left) 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: snap_remove snap 49 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: finishing write tid 3 to mira0437250-456 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: finishing write tid 5 to mira0437250-456 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: finishing write tid 6 to mira0437250-456 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 456 v 531 (ObjNum 808 snap 130 seq_num 808) dirty exists 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: finishing write tid 1 to mira0437250-410 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: finishing write tid 3 to mira0437250-410 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: finishing write tid 4 to mira0437250-410 2014-07-01 16:58:45,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 476 (ObjNum 809 snap 130 seq_num 809) dirty exists 2014-07-01 16:58:45,869.869 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 16:58:46,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: done (6 left) 2014-07-01 16:58:46,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: done (5 left) 2014-07-01 16:58:46,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:2863: done (4 left) 2014-07-01 16:58:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: done (3 left) 2014-07-01 16:58:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: done (2 left) 2014-07-01 16:58:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:2866: done (1 left) 2014-07-01 16:58:46,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: done (0 left) 2014-07-01 16:58:46,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2868: read oid 495 snap -1 2014-07-01 16:58:46,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2869: read oid 291 snap -1 2014-07-01 16:58:46,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: read oid 261 snap -1 2014-07-01 16:58:46,046.046 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 487 current snap is 0 2014-07-01 16:58:46,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: seq_num 1806 ranges {1716224=716800} 2014-07-01 16:58:46,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: writing mira0435601-487 from 1716224 to 2433024 tid 1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2872: read oid 434 snap -1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: read oid 274 snap -1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 489 current snap is 0 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: seq_num 1807 ranges {5861376=745472,6606848=524288,7131136=442368,7573504=237568} 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: writing mira0435601-489 from 5861376 to 6606848 tid 1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: writing mira0435601-489 from 6606848 to 7131136 tid 2 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: writing mira0435601-489 from 7131136 to 7573504 tid 3 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: writing mira0435601-489 from 7573504 to 7811072 tid 4 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: seq_num 1808 ranges {0=262144} 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: writing mira0435601-296 from 0 to 262144 tid 1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2876: read oid 480 snap -1 2014-07-01 16:58:46,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2869: expect (ObjNum 1114 snap 0 seq_num 1114) 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1337: done (4 left) 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1339: done (3 left) 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1341: done (2 left) 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: write oid 300 current snap is 130 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: seq_num 811 ranges {122967=1} 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: writing mira0437250-300 from 122967 to 122968 tid 1 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: write oid 327 current snap is 130 2014-07-01 16:58:46,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: seq_num 812 ranges {583126=429190,1383125=1} 2014-07-01 16:58:46,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: writing mira0437250-327 from 583126 to 1012316 tid 1 2014-07-01 16:58:46,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: writing mira0437250-327 from 1383125 to 1383126 tid 2 2014-07-01 16:58:46,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: write oid 486 current snap is 130 2014-07-01 16:58:46,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: seq_num 813 ranges {527956=553442,1812658=315298} 2014-07-01 16:58:46,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0437250-486 from 527956 to 1081398 tid 1 2014-07-01 16:58:46,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: writing mira0437250-486 from 1812658 to 2127956 tid 2 2014-07-01 16:58:46,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: read oid 274 snap 80 2014-07-01 16:58:46,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 1 to mira0437250-300 2014-07-01 16:58:46,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 3 to mira0437250-300 2014-07-01 16:58:46,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: finishing write tid 4 to mira0437250-300 2014-07-01 16:58:46,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 630 (ObjNum 811 snap 130 seq_num 811) dirty exists 2014-07-01 16:58:46,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:1342: done (5 left) 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1346: snap_remove snap 117 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 1 to mira0437250-327 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 2 to mira0437250-327 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 4 to mira0437250-327 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: finishing write tid 5 to mira0437250-327 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 353 (ObjNum 812 snap 130 seq_num 812) dirty exists 2014-07-01 16:58:46,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: expect (ObjNum 635 snap 60 seq_num 635) 2014-07-01 16:58:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:1338: expect (ObjNum 254 snap 0 seq_num 254) 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: finishing write tid 1 to mira0437250-75 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: finishing write tid 2 to mira0437250-75 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: finishing write tid 3 to mira0437250-75 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: finishing write tid 5 to mira0437250-75 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: finishing write tid 6 to mira0437250-75 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 242 (ObjNum 806 snap 130 seq_num 806) dirty exists 2014-07-01 16:58:46,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 1 to mira0437250-486 2014-07-01 16:58:46,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 2 to mira0437250-486 2014-07-01 16:58:46,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 4 to mira0437250-486 2014-07-01 16:58:46,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: finishing write tid 5 to mira0437250-486 2014-07-01 16:58:46,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 246 (ObjNum 813 snap 130 seq_num 813) dirty exists 2014-07-01 16:58:47,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1334: done (5 left) 2014-07-01 16:58:47,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1338: done (4 left) 2014-07-01 16:58:47,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1343: done (3 left) 2014-07-01 16:58:47,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1344: done (2 left) 2014-07-01 16:58:47,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1345: done (1 left) 2014-07-01 16:58:47,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1346: done (0 left) 2014-07-01 16:58:47,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: read oid 145 snap 91 2014-07-01 16:58:47,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: rollback oid 431 current snap is 130 2014-07-01 16:58:47,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 431 to 122 2014-07-01 16:58:47,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: read oid 390 snap -1 2014-07-01 16:58:47,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: write oid 199 current snap is 130 2014-07-01 16:58:47,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: seq_num 814 ranges {624210=408894,1526790=297420} 2014-07-01 16:58:47,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: writing mira0437250-199 from 624210 to 1033104 tid 1 2014-07-01 16:58:47,269.269 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 16:58:47,278.278 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 16:58:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: writing mira0437250-199 from 1526790 to 1824210 tid 2 2014-07-01 16:58:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: write oid 405 current snap is 130 2014-07-01 16:58:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: seq_num 815 ranges {568015=532123,1639898=431139,2528488=39527} 2014-07-01 16:58:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: writing mira0437250-405 from 568015 to 1100138 tid 1 2014-07-01 16:58:47,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: writing mira0437250-405 from 1639898 to 2071037 tid 2 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: writing mira0437250-405 from 2528488 to 2568015 tid 3 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: write oid 437 current snap is 130 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: seq_num 816 ranges {629197=752503,1932462=652758,3210993=618204} 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0437250-437 from 629197 to 1381700 tid 1 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0437250-437 from 1932462 to 2585220 tid 2 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: writing mira0437250-437 from 3210993 to 3829197 tid 3 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1353: snap_remove snap 128 2014-07-01 16:58:47,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: expect (ObjNum 144 snap 0 seq_num 144) 2014-07-01 16:58:48,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2868: expect (ObjNum 1633 snap 0 seq_num 1633) 2014-07-01 16:58:48,308.308 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 16:58:48,323.323 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 16:58:48,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 1 to mira0437250-437 2014-07-01 16:58:48,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 2 to mira0437250-437 2014-07-01 16:58:48,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 3 to mira0437250-437 2014-07-01 16:58:48,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 5 to mira0437250-437 2014-07-01 16:58:48,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: finishing write tid 6 to mira0437250-437 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 488 (ObjNum 816 snap 130 seq_num 816) dirty exists 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 357 (ObjNum 430 snap 0 seq_num 430) dirty exists 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: finishing write tid 1 to mira0437250-199 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: finishing write tid 2 to mira0437250-199 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: finishing write tid 4 to mira0437250-199 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: finishing write tid 5 to mira0437250-199 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 384 (ObjNum 814 snap 130 seq_num 814) dirty exists 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: finishing write tid 1 to mira0437250-405 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: finishing write tid 2 to mira0437250-405 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: finishing write tid 3 to mira0437250-405 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: finishing write tid 5 to mira0437250-405 2014-07-01 16:58:48,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: finishing write tid 6 to mira0437250-405 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 382 (ObjNum 815 snap 130 seq_num 815) dirty exists 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1347: done (6 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1348: done (5 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1349: done (4 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1350: done (3 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1351: done (2 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1352: done (1 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1353: done (0 left) 2014-07-01 16:58:48,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1354: delete oid 111 current snap is 130 2014-07-01 16:58:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1354: done (0 left) 2014-07-01 16:58:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: rollback oid 353 current snap is 130 2014-07-01 16:58:48,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 353 to 115 2014-07-01 16:58:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: read oid 196 snap -1 2014-07-01 16:58:48,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1357: delete oid 144 current snap is 130 2014-07-01 16:58:48,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:1357: done (2 left) 2014-07-01 16:58:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: rollback oid 73 current snap is 130 2014-07-01 16:58:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 73 to 118 2014-07-01 16:58:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1359: rollback oid 225 current snap is 130 2014-07-01 16:58:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 225 to 125 2014-07-01 16:58:48,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: write oid 292 current snap is 130 2014-07-01 16:58:48,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: seq_num 819 ranges {613062=400000} 2014-07-01 16:58:48,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: writing mira0437250-292 from 613062 to 1013062 tid 1 2014-07-01 16:58:48,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: read oid 243 snap 111 2014-07-01 16:58:48,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: expect (ObjNum 750 snap 109 seq_num 750) 2014-07-01 16:58:49,285.285 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 16:58:49,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 16:58:49,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 240 (ObjNum 352 snap 0 seq_num 352) dirty exists 2014-07-01 16:58:49,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 280 (ObjNum 72 snap 0 seq_num 72) dirty exists 2014-07-01 16:58:49,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 0 (ObjNum 22289328 snap 0 seq_num 0) dirty dne 2014-07-01 16:58:49,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: expect (ObjNum 585 snap 35 seq_num 585) 2014-07-01 16:58:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1355: done (5 left) 2014-07-01 16:58:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1356: done (4 left) 2014-07-01 16:58:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1358: done (3 left) 2014-07-01 16:58:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1359: done (2 left) 2014-07-01 16:58:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1361: done (1 left) 2014-07-01 16:58:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: delete oid 190 current snap is 130 2014-07-01 16:58:50,312.312 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 16:58:50,321.321 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 16:58:50,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: finishing write tid 1 to mira0435601-487 2014-07-01 16:58:50,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: finishing write tid 3 to mira0435601-487 2014-07-01 16:58:50,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: finishing write tid 4 to mira0435601-487 2014-07-01 16:58:50,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 194 (ObjNum 1806 snap 0 seq_num 1806) dirty exists 2014-07-01 16:58:50,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2872: expect (ObjNum 1502 snap 0 seq_num 1502) 2014-07-01 16:58:51,330.330 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 16:58:51,341.341 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 16:58:51,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: finishing write tid 1 to mira0437250-292 2014-07-01 16:58:51,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: finishing write tid 3 to mira0437250-292 2014-07-01 16:58:51,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: finishing write tid 4 to mira0437250-292 2014-07-01 16:58:51,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 657 (ObjNum 819 snap 130 seq_num 819) dirty exists 2014-07-01 16:58:52,357.357 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 16:58:52,363.363 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 16:58:52,364.364 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:58:53,263.263 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:58:53,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: finishing write tid 1 to mira0435601-296 2014-07-01 16:58:53,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: finishing write tid 3 to mira0435601-296 2014-07-01 16:58:53,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: finishing write tid 4 to mira0435601-296 2014-07-01 16:58:53,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 215 (ObjNum 1808 snap 0 seq_num 1808) dirty exists 2014-07-01 16:58:53,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: expect (ObjNum 1237 snap 0 seq_num 1237) 2014-07-01 16:58:54,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: expect (ObjNum 1249 snap 0 seq_num 1249) 2014-07-01 16:58:56,143.143 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:58:57,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2876: expect (ObjNum 1588 snap 0 seq_num 1588) 2014-07-01 16:58:57,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 1 to mira0435601-489 2014-07-01 16:58:57,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 2 to mira0435601-489 2014-07-01 16:58:57,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 3 to mira0435601-489 2014-07-01 16:58:57,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 4 to mira0435601-489 2014-07-01 16:58:57,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 6 to mira0435601-489 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: finishing write tid 7 to mira0435601-489 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 302 (ObjNum 1807 snap 0 seq_num 1807) dirty exists 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2868: done (8 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2869: done (7 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: done (6 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: done (5 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2872: done (4 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: done (3 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: done (2 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: done (1 left) 2014-07-01 16:58:57,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:2876: done (0 left) 2014-07-01 16:58:57,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 16:58:57,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: seq_num 1809 ranges {3604480=135168} 2014-07-01 16:58:57,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: writing mira0435601-31 from 3604480 to 3739648 tid 1 2014-07-01 16:58:57,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: read oid 486 snap -1 2014-07-01 16:58:57,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: delete oid 454 current snap is 0 2014-07-01 16:58:57,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: expect (ObjNum 1419 snap 0 seq_num 1419) 2014-07-01 16:58:57,581.581 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:58:59,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: finishing write tid 1 to mira0435601-31 2014-07-01 16:58:59,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: finishing write tid 3 to mira0435601-31 2014-07-01 16:58:59,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: finishing write tid 4 to mira0435601-31 2014-07-01 16:58:59,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 276 (ObjNum 1809 snap 0 seq_num 1809) dirty exists 2014-07-01 16:58:59,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: done (2 left) 2014-07-01 16:58:59,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: done (1 left) 2014-07-01 16:58:59,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: done (0 left) 2014-07-01 16:58:59,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: read oid 268 snap -1 2014-07-01 16:58:59,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 233 current snap is 0 2014-07-01 16:58:59,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: seq_num 1811 ranges {3055616=188416} 2014-07-01 16:58:59,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: writing mira0435601-233 from 3055616 to 3244032 tid 1 2014-07-01 16:58:59,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 298 current snap is 0 2014-07-01 16:58:59,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: seq_num 1812 ranges {4046848=708608,4755456=401408} 2014-07-01 16:58:59,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: writing mira0435601-298 from 4046848 to 4755456 tid 1 2014-07-01 16:58:59,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: writing mira0435601-298 from 4755456 to 5156864 tid 2 2014-07-01 16:58:59,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2883: read oid 273 snap -1 2014-07-01 16:58:59,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: expect (ObjNum 1726 snap 0 seq_num 1726) 2014-07-01 16:59:00,594.594 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:01,296.296 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:58:44.725056', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881347896, u'health': u'HEALTH_OK', u'kb_used': 15433156, u'store_stats': {u'bytes_total': 10760114, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4468642}}, {u'last_updated': u'2014-07-01 07:58:54.542720', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885338132, u'health': u'HEALTH_OK', u'kb_used': 11442920, u'store_stats': {u'bytes_total': 14954784, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469008}}, {u'last_updated': u'2014-07-01 07:58:11.324942', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885509600, u'health': u'HEALTH_OK', u'kb_used': 11271452, u'store_stats': {u'bytes_total': 9711957, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'16 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'19 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 694/3204 objects degraded (21.660%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 6633578, u'degraded_objects': 694, u'num_pgs': 444, u'recovering_keys_per_sec': 0, u'data_bytes': 2560135273, u'degraded_total': 3204, u'bytes_used': 6197338112, u'recovering_objects_per_sec': 11, u'version': 2107, u'pgs_by_state': [{u'count': 19, u'state_name': u'active+recovery_wait'}, {u'count': 409, u'state_name': u'active+clean'}, {u'count': 16, u'state_name': u'active+recovering'}], u'degraded_ratio': u'21.660', u'bytes_avail': 5992101707776}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 901, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:59:01,297.297 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:02,031.031 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:59:02,032.032 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:02,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: finishing write tid 1 to mira0435601-298 2014-07-01 16:59:02,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: finishing write tid 2 to mira0435601-298 2014-07-01 16:59:02,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: finishing write tid 4 to mira0435601-298 2014-07-01 16:59:02,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: finishing write tid 5 to mira0435601-298 2014-07-01 16:59:02,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 150 (ObjNum 1812 snap 0 seq_num 1812) dirty exists 2014-07-01 16:59:02,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: done (3 left) 2014-07-01 16:59:02,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: done (2 left) 2014-07-01 16:59:02,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2883: done (1 left) 2014-07-01 16:59:02,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 282 current snap is 0 2014-07-01 16:59:02,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: seq_num 1813 ranges {5574656=782336} 2014-07-01 16:59:02,814.814 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:02,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: writing mira0435601-282 from 5574656 to 6356992 tid 1 2014-07-01 16:59:02,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2885: delete oid 50 current snap is 0 2014-07-01 16:59:02,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: finishing write tid 1 to mira0435601-233 2014-07-01 16:59:02,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: finishing write tid 3 to mira0435601-233 2014-07-01 16:59:02,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: finishing write tid 4 to mira0435601-233 2014-07-01 16:59:02,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 233 v 150 (ObjNum 1811 snap 0 seq_num 1811) dirty exists 2014-07-01 16:59:03,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: finishing write tid 1 to mira0435601-282 2014-07-01 16:59:03,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: finishing write tid 3 to mira0435601-282 2014-07-01 16:59:03,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: done (2 left) 2014-07-01 16:59:03,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2885: done (1 left) 2014-07-01 16:59:03,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2886: read oid 429 snap -1 2014-07-01 16:59:03,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 473 current snap is 0 2014-07-01 16:59:03,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: seq_num 1815 ranges {1843200=528384} 2014-07-01 16:59:03,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: writing mira0435601-473 from 1843200 to 2371584 tid 1 2014-07-01 16:59:03,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: delete oid 44 current snap is 0 2014-07-01 16:59:03,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: finishing write tid 4 to mira0435601-282 2014-07-01 16:59:03,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 304 (ObjNum 1813 snap 0 seq_num 1813) dirty exists 2014-07-01 16:59:03,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:2886: expect (ObjNum 428 snap 0 seq_num 428) 2014-07-01 16:59:04,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: finishing write tid 1 to mira0435601-473 2014-07-01 16:59:04,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: finishing write tid 3 to mira0435601-473 2014-07-01 16:59:04,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: finishing write tid 4 to mira0435601-473 2014-07-01 16:59:04,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 238 (ObjNum 1815 snap 0 seq_num 1815) dirty exists 2014-07-01 16:59:04,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: done (3 left) 2014-07-01 16:59:04,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2886: done (2 left) 2014-07-01 16:59:04,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: done (1 left) 2014-07-01 16:59:04,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: done (0 left) 2014-07-01 16:59:04,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 437 current snap is 0 2014-07-01 16:59:04,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: seq_num 1817 ranges {0=73728} 2014-07-01 16:59:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: writing mira0435601-437 from 0 to 73728 tid 1 2014-07-01 16:59:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2890: read oid 119 snap -1 2014-07-01 16:59:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: read oid 334 snap -1 2014-07-01 16:59:04,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: read oid 159 snap -1 2014-07-01 16:59:04,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 189 current snap is 0 2014-07-01 16:59:04,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: seq_num 1818 ranges {3506176=749568,4255744=532480,4788224=512000,5300224=561152} 2014-07-01 16:59:04,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: writing mira0435601-189 from 3506176 to 4255744 tid 1 2014-07-01 16:59:04,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: writing mira0435601-189 from 4255744 to 4788224 tid 2 2014-07-01 16:59:04,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: writing mira0435601-189 from 4788224 to 5300224 tid 3 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: writing mira0435601-189 from 5300224 to 5861376 tid 4 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 92 current snap is 0 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: seq_num 1819 ranges {1003520=774144,1777664=401408} 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: writing mira0435601-92 from 1003520 to 1777664 tid 1 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: writing mira0435601-92 from 1777664 to 2179072 tid 2 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2895: read oid 473 snap -1 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: finishing write tid 1 to mira0435601-437 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: finishing write tid 3 to mira0435601-437 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2896: delete oid 272 current snap is 0 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2896: done (7 left) 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 493 current snap is 0 2014-07-01 16:59:04,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: seq_num 1821 ranges {90112=520192,610304=401408} 2014-07-01 16:59:04,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: writing mira0435601-493 from 90112 to 610304 tid 1 2014-07-01 16:59:04,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: writing mira0435601-493 from 610304 to 1011712 tid 2 2014-07-01 16:59:04,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: read oid 82 snap -1 2014-07-01 16:59:04,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2890: expect (ObjNum 1329 snap 0 seq_num 1329) 2014-07-01 16:59:04,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:1360: done (1 left) 2014-07-01 16:59:04,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1362: done (0 left) 2014-07-01 16:59:04,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: write oid 161 current snap is 130 2014-07-01 16:59:04,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: seq_num 821 ranges {769474=552517,1969473=1} 2014-07-01 16:59:04,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: writing mira0437250-161 from 769474 to 1321991 tid 1 2014-07-01 16:59:04,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: writing mira0437250-161 from 1969473 to 1969474 tid 2 2014-07-01 16:59:04,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: read oid 217 snap -1 2014-07-01 16:59:04,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:1365: snap_remove snap 101 2014-07-01 16:59:04,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: expect (ObjNum 638 snap 64 seq_num 638) 2014-07-01 16:59:06,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: finishing write tid 1 to mira0437250-161 2014-07-01 16:59:06,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: finishing write tid 2 to mira0437250-161 2014-07-01 16:59:06,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: finishing write tid 4 to mira0437250-161 2014-07-01 16:59:06,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: finishing write tid 5 to mira0437250-161 2014-07-01 16:59:06,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 665 (ObjNum 821 snap 130 seq_num 821) dirty exists 2014-07-01 16:59:06,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1363: done (2 left) 2014-07-01 16:59:06,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1364: done (1 left) 2014-07-01 16:59:06,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1365: done (0 left) 2014-07-01 16:59:06,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: write oid 474 current snap is 130 2014-07-01 16:59:06,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: seq_num 822 ranges {713280=407969,1877288=709001,2713279=1} 2014-07-01 16:59:06,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: writing mira0437250-474 from 713280 to 1121249 tid 1 2014-07-01 16:59:06,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: writing mira0437250-474 from 1877288 to 2586289 tid 2 2014-07-01 16:59:06,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: writing mira0437250-474 from 2713279 to 2713280 tid 3 2014-07-01 16:59:06,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:1367: snap_create 2014-07-01 16:59:06,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: finishing write tid 1 to mira0437250-474 2014-07-01 16:59:06,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: finishing write tid 2 to mira0437250-474 2014-07-01 16:59:06,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: finishing write tid 3 to mira0437250-474 2014-07-01 16:59:06,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: finishing write tid 5 to mira0437250-474 2014-07-01 16:59:06,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: finishing write tid 6 to mira0437250-474 2014-07-01 16:59:06,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 372 (ObjNum 822 snap 130 seq_num 822) dirty exists 2014-07-01 16:59:06,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1366: done (1 left) 2014-07-01 16:59:06,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1367: done (0 left) 2014-07-01 16:59:06,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1368: rollback oid 495 current snap is 131 2014-07-01 16:59:06,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 495 to 115 2014-07-01 16:59:06,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: read oid 40 snap -1 2014-07-01 16:59:06,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:1370: snap_remove snap 104 2014-07-01 16:59:06,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 445 (ObjNum 494 snap 0 seq_num 494) dirty exists 2014-07-01 16:59:07,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: expect (ObjNum 743 snap 108 seq_num 743) 2014-07-01 16:59:07,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: expect (ObjNum 158 snap 0 seq_num 158) 2014-07-01 16:59:08,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:1368: done (2 left) 2014-07-01 16:59:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:1369: done (1 left) 2014-07-01 16:59:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:1370: done (0 left) 2014-07-01 16:59:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: snap_create 2014-07-01 16:59:08,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 1 to mira0435601-189 2014-07-01 16:59:08,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 2 to mira0435601-189 2014-07-01 16:59:08,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 3 to mira0435601-189 2014-07-01 16:59:08,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: finishing write tid 4 to mira0435601-437 2014-07-01 16:59:08,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 260 (ObjNum 1817 snap 0 seq_num 1817) dirty exists 2014-07-01 16:59:08,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: expect (ObjNum 532 snap 0 seq_num 532) 2014-07-01 16:59:08,867.867 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:09,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:1371: done (0 left) 2014-07-01 16:59:09,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: write oid 338 current snap is 132 2014-07-01 16:59:09,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: seq_num 823 ranges {409533=531197,1591419=530620,2409532=1} 2014-07-01 16:59:09,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: writing mira0437250-338 from 409533 to 940730 tid 1 2014-07-01 16:59:09,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: writing mira0437250-338 from 1591419 to 2122039 tid 2 2014-07-01 16:59:09,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: writing mira0437250-338 from 2409532 to 2409533 tid 3 2014-07-01 16:59:09,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: write oid 246 current snap is 132 2014-07-01 16:59:09,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: seq_num 824 ranges {754363=504026,1671504=600815,2784445=369918} 2014-07-01 16:59:09,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: writing mira0437250-246 from 754363 to 1258389 tid 1 2014-07-01 16:59:09,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: writing mira0437250-246 from 1671504 to 2272319 tid 2 2014-07-01 16:59:09,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: writing mira0437250-246 from 2784445 to 3154363 tid 3 2014-07-01 16:59:09,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: write oid 34 current snap is 132 2014-07-01 16:59:09,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: seq_num 825 ranges {564921=627254,1650340=671011,2850375=514546} 2014-07-01 16:59:09,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: writing mira0437250-34 from 564921 to 1192175 tid 1 2014-07-01 16:59:09,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: writing mira0437250-34 from 1650340 to 2321351 tid 2 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: writing mira0437250-34 from 2850375 to 3364921 tid 3 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: write oid 217 current snap is 132 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: seq_num 826 ranges {109750=1} 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: writing mira0437250-217 from 109750 to 109751 tid 1 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1376: read oid 384 snap -1 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: write oid 495 current snap is 132 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: seq_num 827 ranges {453556=400000} 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: writing mira0437250-495 from 453556 to 853556 tid 1 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: read oid 263 snap 107 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: rollback oid 336 current snap is 132 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 336 to 118 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: snap_create 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 0 (ObjNum 37058000 snap 0 seq_num 44855824) dirty dne 2014-07-01 16:59:09,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1376: expect (ObjNum 383 snap 0 seq_num 383) 2014-07-01 16:59:09,637.637 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:09,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2895: expect (ObjNum 1815 snap 0 seq_num 1815) 2014-07-01 16:59:10,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 1 to mira0437250-495 2014-07-01 16:59:10,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 3 to mira0437250-495 2014-07-01 16:59:10,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: finishing write tid 4 to mira0437250-495 2014-07-01 16:59:10,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 454 (ObjNum 827 snap 132 seq_num 827) dirty exists 2014-07-01 16:59:10,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: finishing write tid 1 to mira0437250-246 2014-07-01 16:59:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: finishing write tid 2 to mira0437250-246 2014-07-01 16:59:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: finishing write tid 3 to mira0437250-246 2014-07-01 16:59:10,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: finishing write tid 5 to mira0437250-246 2014-07-01 16:59:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: finishing write tid 1 to mira0437250-338 2014-07-01 16:59:10,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: expect (ObjNum 262 snap 0 seq_num 262) 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: finishing write tid 2 to mira0437250-338 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: finishing write tid 3 to mira0437250-338 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: finishing write tid 1 to mira0437250-34 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: finishing write tid 2 to mira0437250-34 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: finishing write tid 3 to mira0437250-34 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: finishing write tid 5 to mira0437250-338 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: finishing write tid 5 to mira0437250-34 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: finishing write tid 6 to mira0437250-246 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 246 v 576 (ObjNum 824 snap 132 seq_num 824) dirty exists 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: finishing write tid 6 to mira0437250-338 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 410 (ObjNum 823 snap 132 seq_num 823) dirty exists 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: finishing write tid 6 to mira0437250-34 2014-07-01 16:59:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 368 (ObjNum 825 snap 132 seq_num 825) dirty exists 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1372: done (8 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1373: done (7 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1374: done (6 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1376: done (5 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1377: done (4 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1378: done (3 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1379: done (2 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1380: done (1 left) 2014-07-01 16:59:10,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: delete oid 387 current snap is 133 2014-07-01 16:59:10,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1381: done (1 left) 2014-07-01 16:59:10,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: snap_create 2014-07-01 16:59:11,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 4 to mira0435601-189 2014-07-01 16:59:11,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 6 to mira0435601-189 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 7 to mira0435601-189 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 262 (ObjNum 1818 snap 0 seq_num 1818) dirty exists 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: finishing write tid 1 to mira0435601-493 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: finishing write tid 2 to mira0435601-493 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: finishing write tid 4 to mira0435601-493 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: finishing write tid 5 to mira0435601-493 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 286 (ObjNum 1821 snap 0 seq_num 1821) dirty exists 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: finishing write tid 1 to mira0435601-92 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: finishing write tid 2 to mira0435601-92 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: finishing write tid 4 to mira0435601-92 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: finishing write tid 5 to mira0435601-92 2014-07-01 16:59:11,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 261 (ObjNum 1819 snap 0 seq_num 1819) dirty exists 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: done (8 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2890: done (7 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: done (6 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: done (5 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: done (4 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: done (3 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2895: done (2 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: done (1 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: done (0 left) 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: read oid 383 snap -1 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 263 current snap is 0 2014-07-01 16:59:11,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: seq_num 1822 ranges {2600960=593920} 2014-07-01 16:59:11,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: writing mira0435601-263 from 2600960 to 3194880 tid 1 2014-07-01 16:59:11,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2901: read oid 148 snap -1 2014-07-01 16:59:11,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2902: read oid 266 snap -1 2014-07-01 16:59:11,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: read oid 92 snap -1 2014-07-01 16:59:11,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 206 current snap is 0 2014-07-01 16:59:11,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: seq_num 1823 ranges {3682304=466944} 2014-07-01 16:59:11,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1382: done (1 left) 2014-07-01 16:59:11,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: write oid 300 current snap is 134 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: seq_num 829 ranges {494640=550567,1294639=1} 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: writing mira0437250-300 from 494640 to 1045207 tid 1 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: writing mira0437250-300 from 1294639 to 1294640 tid 2 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: read oid 261 snap -1 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: write oid 176 current snap is 134 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: seq_num 830 ranges {554797=406020,1579208=559907,2554796=1} 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: writing mira0437250-176 from 554797 to 960817 tid 1 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: writing mira0437250-176 from 1579208 to 2139115 tid 2 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: writing mira0437250-176 from 2554796 to 2554797 tid 3 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: write oid 293 current snap is 134 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: writing mira0435601-206 from 3682304 to 4149248 tid 1 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 293 current snap is 0 2014-07-01 16:59:11,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: seq_num 1824 ranges {2732032=495616,3227648=524288,3751936=503808,4255744=176128} 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: seq_num 831 ranges {651051=530272,1962140=288911} 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: writing mira0437250-293 from 651051 to 1181323 tid 1 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: writing mira0437250-293 from 1962140 to 2251051 tid 2 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1387: snap_create 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: finishing write tid 1 to mira0437250-176 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: finishing write tid 2 to mira0437250-176 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: finishing write tid 3 to mira0437250-176 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: finishing write tid 5 to mira0437250-176 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: finishing write tid 6 to mira0437250-176 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 600 (ObjNum 830 snap 134 seq_num 830) dirty exists 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 1 to mira0437250-293 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 2 to mira0437250-293 2014-07-01 16:59:11,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 4 to mira0437250-293 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: finishing write tid 5 to mira0437250-293 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 397 (ObjNum 831 snap 134 seq_num 831) dirty exists 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: writing mira0435601-293 from 2732032 to 3227648 tid 1 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: writing mira0435601-293 from 3227648 to 3751936 tid 2 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: writing mira0435601-293 from 3751936 to 4255744 tid 3 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: writing mira0435601-293 from 4255744 to 4431872 tid 4 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 262 current snap is 0 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: seq_num 1825 ranges {5464064=765952,6230016=507904,6737920=294912} 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: writing mira0435601-262 from 5464064 to 6230016 tid 1 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: writing mira0435601-262 from 6230016 to 6737920 tid 2 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: writing mira0435601-262 from 6737920 to 7032832 tid 3 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 395 current snap is 0 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: seq_num 1826 ranges {516096=688128,1204224=737280,1941504=450560,2392064=417792} 2014-07-01 16:59:11,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: writing mira0435601-395 from 516096 to 1204224 tid 1 2014-07-01 16:59:11,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: writing mira0435601-395 from 1204224 to 1941504 tid 2 2014-07-01 16:59:11,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: writing mira0435601-395 from 1941504 to 2392064 tid 3 2014-07-01 16:59:11,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: writing mira0435601-395 from 2392064 to 2809856 tid 4 2014-07-01 16:59:11,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:2908: read oid 428 snap -1 2014-07-01 16:59:11,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: expect (ObjNum 957 snap 0 seq_num 957) 2014-07-01 16:59:12,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1385: done (5 left) 2014-07-01 16:59:12,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1386: done (4 left) 2014-07-01 16:59:12,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1387: done (3 left) 2014-07-01 16:59:12,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:1388: snap_remove snap 107 2014-07-01 16:59:12,813.813 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:13,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1388: done (3 left) 2014-07-01 16:59:13,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1389: snap_remove snap 130 2014-07-01 16:59:13,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: finishing write tid 1 to mira0435601-263 2014-07-01 16:59:13,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: finishing write tid 3 to mira0435601-263 2014-07-01 16:59:13,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: expect (ObjNum 1819 snap 0 seq_num 1819) 2014-07-01 16:59:13,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: expect (ObjNum 260 snap 0 seq_num 260) 2014-07-01 16:59:13,502.502 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:58:44.725056', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881347896, u'health': u'HEALTH_OK', u'kb_used': 15433156, u'store_stats': {u'bytes_total': 10760114, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4468642}}, {u'last_updated': u'2014-07-01 07:58:54.542720', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885338132, u'health': u'HEALTH_OK', u'kb_used': 11442920, u'store_stats': {u'bytes_total': 14954784, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469008}}, {u'last_updated': u'2014-07-01 07:59:11.325340', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885315256, u'health': u'HEALTH_OK', u'kb_used': 11465796, u'store_stats': {u'bytes_total': 17051989, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 286/3211 objects degraded (8.907%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 286, u'num_pgs': 444, u'data_bytes': 2569990580, u'degraded_total': 3211, u'bytes_used': 6083911680, u'version': 2120, u'pgs_by_state': [{u'count': 433, u'state_name': u'active+clean'}, {u'count': 11, u'state_name': u'active+recovering'}], u'degraded_ratio': u'8.907', u'bytes_avail': 5992215134208}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 909, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:59:13,503.503 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:13,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: finishing write tid 1 to mira0437250-300 2014-07-01 16:59:13,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: finishing write tid 2 to mira0437250-300 2014-07-01 16:59:13,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: finishing write tid 4 to mira0437250-300 2014-07-01 16:59:14,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:1384: done (3 left) 2014-07-01 16:59:14,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1389: done (2 left) 2014-07-01 16:59:14,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1390: snap_create 2014-07-01 16:59:14,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 1 to mira0437250-217 2014-07-01 16:59:14,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 3 to mira0437250-217 2014-07-01 16:59:14,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: finishing write tid 4 to mira0437250-217 2014-07-01 16:59:14,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 495 (ObjNum 826 snap 132 seq_num 826) dirty exists 2014-07-01 16:59:14,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: finishing write tid 4 to mira0435601-263 2014-07-01 16:59:14,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 261 (ObjNum 1822 snap 0 seq_num 1822) dirty exists 2014-07-01 16:59:14,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: finishing write tid 1 to mira0435601-206 2014-07-01 16:59:14,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: finishing write tid 3 to mira0435601-206 2014-07-01 16:59:14,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: finishing write tid 4 to mira0435601-206 2014-07-01 16:59:14,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 232 (ObjNum 1823 snap 0 seq_num 1823) dirty exists 2014-07-01 16:59:14,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 1 to mira0435601-395 2014-07-01 16:59:14,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 2 to mira0435601-395 2014-07-01 16:59:14,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 3 to mira0435601-395 2014-07-01 16:59:14,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 4 to mira0435601-395 2014-07-01 16:59:14,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 6 to mira0435601-395 2014-07-01 16:59:14,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: finishing write tid 7 to mira0435601-395 2014-07-01 16:59:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 259 (ObjNum 1826 snap 0 seq_num 1826) dirty exists 2014-07-01 16:59:14,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:2908: expect (ObjNum 1327 snap 0 seq_num 1327) 2014-07-01 16:59:14,253.253 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:59:14,254.254 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:14,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:2902: expect (ObjNum 265 snap 0 seq_num 265) 2014-07-01 16:59:15,029.029 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:15,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1375: done (2 left) 2014-07-01 16:59:15,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:1390: done (1 left) 2014-07-01 16:59:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: rollback oid 244 current snap is 136 2014-07-01 16:59:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 244 to 111 2014-07-01 16:59:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1392: read oid 214 snap -1 2014-07-01 16:59:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: write oid 357 current snap is 136 2014-07-01 16:59:15,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: seq_num 832 ranges {594857=502077,1771377=699273,2994856=1} 2014-07-01 16:59:15,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: writing mira0437250-357 from 594857 to 1096934 tid 1 2014-07-01 16:59:15,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: writing mira0437250-357 from 1771377 to 2470650 tid 2 2014-07-01 16:59:15,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: writing mira0437250-357 from 2994856 to 2994857 tid 3 2014-07-01 16:59:15,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:1394: read oid 390 snap -1 2014-07-01 16:59:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 471 (ObjNum 614 snap 47 seq_num 614) dirty exists 2014-07-01 16:59:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: finishing write tid 1 to mira0437250-357 2014-07-01 16:59:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1391: done (4 left) 2014-07-01 16:59:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1392: done (3 left) 2014-07-01 16:59:15,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: snap_create 2014-07-01 16:59:15,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: finishing write tid 2 to mira0437250-357 2014-07-01 16:59:15,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: finishing write tid 3 to mira0437250-357 2014-07-01 16:59:15,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: finishing write tid 5 to mira0437250-357 2014-07-01 16:59:15,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: finishing write tid 6 to mira0437250-357 2014-07-01 16:59:15,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 610 (ObjNum 832 snap 136 seq_num 832) dirty exists 2014-07-01 16:59:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1393: done (3 left) 2014-07-01 16:59:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1394: done (2 left) 2014-07-01 16:59:16,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1395: done (1 left) 2014-07-01 16:59:16,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: snap_remove snap 129 2014-07-01 16:59:16,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2901: expect (ObjNum 1459 snap 0 seq_num 1459) 2014-07-01 16:59:17,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1396: done (1 left) 2014-07-01 16:59:17,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: rollback oid 110 current snap is 137 2014-07-01 16:59:17,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 110 to 114 2014-07-01 16:59:17,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: write oid 55 current snap is 137 2014-07-01 16:59:17,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: seq_num 833 ranges {538663=625305,1884485=769468,3069329=486261,3738662=1} 2014-07-01 16:59:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0437250-55 from 538663 to 1163968 tid 1 2014-07-01 16:59:17,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0437250-55 from 1884485 to 2653953 tid 2 2014-07-01 16:59:17,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0437250-55 from 3069329 to 3555590 tid 3 2014-07-01 16:59:17,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: writing mira0437250-55 from 3738662 to 3738663 tid 4 2014-07-01 16:59:17,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: write oid 337 current snap is 137 2014-07-01 16:59:17,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: seq_num 834 ranges {751268=480757,1715992=439663,2588953=772027,3951267=1} 2014-07-01 16:59:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: writing mira0437250-337 from 751268 to 1232025 tid 1 2014-07-01 16:59:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: writing mira0437250-337 from 1715992 to 2155655 tid 2 2014-07-01 16:59:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: writing mira0437250-337 from 2588953 to 3360980 tid 3 2014-07-01 16:59:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: writing mira0437250-337 from 3951267 to 3951268 tid 4 2014-07-01 16:59:17,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: snap_create 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: finishing write tid 1 to mira0435601-262 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: finishing write tid 2 to mira0435601-262 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: finishing write tid 3 to mira0435601-262 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: finishing write tid 5 to mira0435601-262 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 1 to mira0435601-293 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 2 to mira0435601-293 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 3 to mira0435601-293 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 4 to mira0435601-293 2014-07-01 16:59:17,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 6 to mira0435601-293 2014-07-01 16:59:17,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: done (9 left) 2014-07-01 16:59:17,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: done (8 left) 2014-07-01 16:59:17,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2901: done (7 left) 2014-07-01 16:59:17,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2902: done (6 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: done (5 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: done (4 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: done (3 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2908: done (2 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: read oid 351 snap -1 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: finishing write tid 6 to mira0435601-262 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 262 v 288 (ObjNum 1825 snap 0 seq_num 1825) dirty exists 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: done (2 left) 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 155 current snap is 0 2014-07-01 16:59:17,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: seq_num 1827 ranges {1011712=716800,1728512=401408} 2014-07-01 16:59:17,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: writing mira0435601-155 from 1011712 to 1728512 tid 1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: writing mira0435601-155 from 1728512 to 2129920 tid 2 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2911: read oid 119 snap -1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2912: read oid 7 snap -1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: seq_num 1828 ranges {3321856=585728,3907584=401408} 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: writing mira0435601-397 from 3321856 to 3907584 tid 1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: writing mira0435601-397 from 3907584 to 4308992 tid 2 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: read oid 206 snap -1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2912: done (6 left) 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: read oid 319 snap -1 2014-07-01 16:59:17,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: read oid 64 snap -1 2014-07-01 16:59:17,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:2917: delete oid 283 current snap is 0 2014-07-01 16:59:17,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:2911: expect (ObjNum 1329 snap 0 seq_num 1329) 2014-07-01 16:59:18,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1400: done (4 left) 2014-07-01 16:59:18,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:1401: snap_remove snap 133 2014-07-01 16:59:19,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1401: done (4 left) 2014-07-01 16:59:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: read oid 161 snap -1 2014-07-01 16:59:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: read oid 449 snap -1 2014-07-01 16:59:19,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: delete oid 332 current snap is 138 2014-07-01 16:59:19,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: expect (ObjNum 448 snap 0 seq_num 448) 2014-07-01 16:59:20,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: expect (ObjNum 1550 snap 0 seq_num 1550) 2014-07-01 16:59:20,874.874 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:21,596.596 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:23,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 1 to mira0435601-155 2014-07-01 16:59:23,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 2 to mira0435601-155 2014-07-01 16:59:23,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 4 to mira0435601-155 2014-07-01 16:59:23,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: expect (ObjNum 1731 snap 0 seq_num 1731) 2014-07-01 16:59:24,471.471 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:25,160.160 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:58:44.725056', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881347896, u'health': u'HEALTH_OK', u'kb_used': 15433156, u'store_stats': {u'bytes_total': 10760114, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4468642}}, {u'last_updated': u'2014-07-01 07:58:54.542720', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885338132, u'health': u'HEALTH_OK', u'kb_used': 11442920, u'store_stats': {u'bytes_total': 14954784, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469008}}, {u'last_updated': u'2014-07-01 07:59:11.325340', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885315256, u'health': u'HEALTH_OK', u'kb_used': 11465796, u'store_stats': {u'bytes_total': 17051989, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'8 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 218/3227 objects degraded (6.756%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 6441993, u'degraded_objects': 218, u'num_pgs': 444, u'recovering_keys_per_sec': 0, u'data_bytes': 2582495873, u'degraded_total': 3227, u'bytes_used': 6247108608, u'recovering_objects_per_sec': 2, u'write_bytes_sec': 1203229, u'op_per_sec': 8, u'version': 2131, u'pgs_by_state': [{u'count': 436, u'state_name': u'active+clean'}, {u'count': 8, u'state_name': u'active+recovering'}], u'read_bytes_sec': 6947102, u'degraded_ratio': u'6.756', u'bytes_avail': 5992051937280}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 915, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:59:25,161.161 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:25,861.861 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 16:59:25,861.861 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:26,659.659 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:26,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 454 (ObjNum 109 snap 0 seq_num 109) dirty exists 2014-07-01 16:59:26,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 1 to mira0437250-337 2014-07-01 16:59:27,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 2 to mira0437250-337 2014-07-01 16:59:27,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 3 to mira0437250-337 2014-07-01 16:59:27,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: expect (ObjNum 1823 snap 0 seq_num 1823) 2014-07-01 16:59:27,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 4 to mira0437250-337 2014-07-01 16:59:27,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 6 to mira0437250-337 2014-07-01 16:59:29,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 5 to mira0435601-155 2014-07-01 16:59:29,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 275 (ObjNum 1827 snap 0 seq_num 1827) dirty exists 2014-07-01 16:59:29,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 1 to mira0435601-397 2014-07-01 16:59:29,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 2 to mira0435601-397 2014-07-01 16:59:29,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: finishing write tid 7 to mira0435601-293 2014-07-01 16:59:29,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 192 (ObjNum 1824 snap 0 seq_num 1824) dirty exists 2014-07-01 16:59:29,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: expect (ObjNum 1480 snap 0 seq_num 1480) 2014-07-01 16:59:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: done (8 left) 2014-07-01 16:59:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: done (7 left) 2014-07-01 16:59:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: done (6 left) 2014-07-01 16:59:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2911: done (5 left) 2014-07-01 16:59:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: done (4 left) 2014-07-01 16:59:31,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: done (3 left) 2014-07-01 16:59:31,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: done (2 left) 2014-07-01 16:59:31,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2917: done (1 left) 2014-07-01 16:59:31,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 16:59:31,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: seq_num 1830 ranges {6275072=483328,6758400=720896,7479296=450560,7929856=434176} 2014-07-01 16:59:31,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: writing mira0435601-229 from 6275072 to 6758400 tid 1 2014-07-01 16:59:31,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: writing mira0435601-229 from 6758400 to 7479296 tid 2 2014-07-01 16:59:31,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: writing mira0435601-229 from 7479296 to 7929856 tid 3 2014-07-01 16:59:31,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: writing mira0435601-229 from 7929856 to 8364032 tid 4 2014-07-01 16:59:31,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 274 current snap is 0 2014-07-01 16:59:31,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: seq_num 1831 ranges {1380352=405504} 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: writing mira0435601-274 from 1380352 to 1785856 tid 1 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2920: read oid 97 snap -1 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 1 to mira0435601-229 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 2 to mira0435601-229 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 3 to mira0435601-229 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 289 current snap is 0 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: seq_num 1832 ranges {4947968=278528} 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: writing mira0435601-289 from 4947968 to 5226496 tid 1 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: read oid 194 snap -1 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 4 to mira0435601-229 2014-07-01 16:59:31,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 6 to mira0435601-229 2014-07-01 16:59:31,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: finishing write tid 7 to mira0435601-229 2014-07-01 16:59:31,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 266 (ObjNum 1830 snap 0 seq_num 1830) dirty exists 2014-07-01 16:59:31,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: done (5 left) 2014-07-01 16:59:31,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 110 current snap is 0 2014-07-01 16:59:31,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: seq_num 1833 ranges {2273280=704512,2977792=716800,3694592=86016} 2014-07-01 16:59:31,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: writing mira0435601-110 from 2273280 to 2977792 tid 1 2014-07-01 16:59:31,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: writing mira0435601-110 from 2977792 to 3694592 tid 2 2014-07-01 16:59:31,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: writing mira0435601-110 from 3694592 to 3780608 tid 3 2014-07-01 16:59:31,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: read oid 30 snap -1 2014-07-01 16:59:31,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2925: read oid 138 snap -1 2014-07-01 16:59:31,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2926: delete oid 47 current snap is 0 2014-07-01 16:59:31,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2920: expect (ObjNum 1772 snap 0 seq_num 1772) 2014-07-01 16:59:32,501.501 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:33,269.269 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:36,151.151 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:36,841.841 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:58:44.725056', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881347896, u'health': u'HEALTH_OK', u'kb_used': 15433156, u'store_stats': {u'bytes_total': 10760114, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4468642}}, {u'last_updated': u'2014-07-01 07:58:54.542720', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885338132, u'health': u'HEALTH_OK', u'kb_used': 11442920, u'store_stats': {u'bytes_total': 14954784, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469008}}, {u'last_updated': u'2014-07-01 07:59:11.325340', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885315256, u'health': u'HEALTH_OK', u'kb_used': 11465796, u'store_stats': {u'bytes_total': 17051989, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'8 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 218/3221 objects degraded (6.768%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 218, u'num_pgs': 444, u'data_bytes': 2587261569, u'degraded_total': 3221, u'bytes_used': 6209351680, u'op_per_sec': 3, u'write_bytes_sec': 690744, u'version': 2136, u'pgs_by_state': [{u'count': 436, u'state_name': u'active+clean'}, {u'count': 8, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1679399, u'degraded_ratio': u'6.768', u'bytes_avail': 5992089694208}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 915, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:59:36,842.842 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 1 to mira0435601-289 2014-07-01 16:59:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 3 to mira0435601-289 2014-07-01 16:59:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 4 to mira0435601-289 2014-07-01 16:59:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 232 (ObjNum 1832 snap 0 seq_num 1832) dirty exists 2014-07-01 16:59:37,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 1 to mira0435601-110 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 2 to mira0435601-110 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 3 to mira0435601-110 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 5 to mira0435601-110 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 6 to mira0435601-110 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 175 (ObjNum 1833 snap 0 seq_num 1833) dirty exists 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2920: done (8 left) 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: done (7 left) 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: done (6 left) 2014-07-01 16:59:37,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:2925: done (5 left) 2014-07-01 16:59:37,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:2926: done (4 left) 2014-07-01 16:59:37,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: read oid 400 snap -1 2014-07-01 16:59:37,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 368 current snap is 0 2014-07-01 16:59:37,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: seq_num 1835 ranges {3080192=499712,3579904=479232,4059136=573440,4632576=552960} 2014-07-01 16:59:37,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: writing mira0435601-368 from 3080192 to 3579904 tid 1 2014-07-01 16:59:37,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: writing mira0435601-368 from 3579904 to 4059136 tid 2 2014-07-01 16:59:37,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: writing mira0435601-368 from 4059136 to 4632576 tid 3 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: writing mira0435601-368 from 4632576 to 5185536 tid 4 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 202 current snap is 0 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: seq_num 1836 ranges {0=524288,524288=401408} 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: writing mira0435601-202 from 0 to 524288 tid 1 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: writing mira0435601-202 from 524288 to 925696 tid 2 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2930: read oid 499 snap -1 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2931: delete oid 72 current snap is 0 2014-07-01 16:59:37,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: expect (ObjNum 1477 snap 0 seq_num 1477) 2014-07-01 16:59:37,636.636 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:59:37,636.636 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:38,355.355 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:41,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 1 to mira0435601-202 2014-07-01 16:59:41,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 2 to mira0435601-202 2014-07-01 16:59:41,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 4 to mira0435601-202 2014-07-01 16:59:41,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 5 to mira0435601-202 2014-07-01 16:59:41,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 180 (ObjNum 1836 snap 0 seq_num 1836) dirty exists 2014-07-01 16:59:41,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: done (8 left) 2014-07-01 16:59:41,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: done (7 left) 2014-07-01 16:59:41,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2931: done (6 left) 2014-07-01 16:59:41,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 347 current snap is 0 2014-07-01 16:59:41,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: seq_num 1838 ranges {5050368=671744} 2014-07-01 16:59:41,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: writing mira0435601-347 from 5050368 to 5722112 tid 1 2014-07-01 16:59:41,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2933: read oid 389 snap -1 2014-07-01 16:59:41,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2934: delete oid 466 current snap is 0 2014-07-01 16:59:41,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2930: expect (ObjNum 945 snap 0 seq_num 945) 2014-07-01 16:59:43,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: finishing write tid 1 to mira0435601-347 2014-07-01 16:59:43,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: finishing write tid 3 to mira0435601-347 2014-07-01 16:59:43,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2933: expect (ObjNum 388 snap 0 seq_num 388) 2014-07-01 16:59:44,269.269 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:44,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: finishing write tid 4 to mira0435601-347 2014-07-01 16:59:44,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 264 (ObjNum 1838 snap 0 seq_num 1838) dirty exists 2014-07-01 16:59:44,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2930: done (8 left) 2014-07-01 16:59:44,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: done (7 left) 2014-07-01 16:59:44,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2933: done (6 left) 2014-07-01 16:59:44,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2934: done (5 left) 2014-07-01 16:59:44,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 95 current snap is 0 2014-07-01 16:59:44,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: seq_num 1840 ranges {434176=774144,1208320=692224,1900544=110592} 2014-07-01 16:59:44,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: writing mira0435601-95 from 434176 to 1208320 tid 1 2014-07-01 16:59:44,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: writing mira0435601-95 from 1208320 to 1900544 tid 2 2014-07-01 16:59:44,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: writing mira0435601-95 from 1900544 to 2011136 tid 3 2014-07-01 16:59:44,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:2936: read oid 304 snap -1 2014-07-01 16:59:44,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: read oid 175 snap -1 2014-07-01 16:59:44,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 237 current snap is 0 2014-07-01 16:59:44,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: seq_num 1841 ranges {2363392=90112} 2014-07-01 16:59:44,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: writing mira0435601-237 from 2363392 to 2453504 tid 1 2014-07-01 16:59:44,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2939: read oid 34 snap -1 2014-07-01 16:59:44,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: read oid 286 snap -1 2014-07-01 16:59:44,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2941: delete oid 434 current snap is 0 2014-07-01 16:59:44,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2936: expect (ObjNum 1300 snap 0 seq_num 1300) 2014-07-01 16:59:44,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2939: expect (ObjNum 1586 snap 0 seq_num 1586) 2014-07-01 16:59:44,864.864 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:47,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2936: done (11 left) 2014-07-01 16:59:47,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2939: done (10 left) 2014-07-01 16:59:47,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2941: done (9 left) 2014-07-01 16:59:47,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:2942: read oid 184 snap -1 2014-07-01 16:59:47,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 181 current snap is 0 2014-07-01 16:59:47,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: seq_num 1843 ranges {3559424=143360} 2014-07-01 16:59:47,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: writing mira0435601-181 from 3559424 to 3702784 tid 1 2014-07-01 16:59:47,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2944: delete oid 468 current snap is 0 2014-07-01 16:59:47,970.970 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:48,654.654 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:59:44.725350', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881002872, u'health': u'HEALTH_OK', u'kb_used': 15778180, u'store_stats': {u'bytes_total': 5869066, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4820474}}, {u'last_updated': u'2014-07-01 07:58:54.542720', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885338132, u'health': u'HEALTH_OK', u'kb_used': 11442920, u'store_stats': {u'bytes_total': 14954784, u'bytes_log': 10420224, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469008}}, {u'last_updated': u'2014-07-01 07:59:11.325340', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885315256, u'health': u'HEALTH_OK', u'kb_used': 11465796, u'store_stats': {u'bytes_total': 17051989, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'8 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'4 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 218/3215 objects degraded (6.781%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 218, u'num_pgs': 444, u'data_bytes': 2574846593, u'degraded_total': 3215, u'bytes_used': 6238511104, u'op_per_sec': 2, u'write_bytes_sec': 144179, u'version': 2140, u'pgs_by_state': [{u'count': 436, u'state_name': u'active+clean'}, {u'count': 8, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1153433, u'degraded_ratio': u'6.781', u'bytes_avail': 5992060534784}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 915, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 16:59:48,655.655 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 16:59:49,387.387 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 16:59:49,387.387 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:50,235.235 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:50,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 1 to mira0437250-55 2014-07-01 16:59:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: finishing write tid 5 to mira0437250-300 2014-07-01 16:59:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 707 (ObjNum 829 snap 134 seq_num 829) dirty exists 2014-07-01 16:59:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 2 to mira0437250-55 2014-07-01 16:59:50,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 4 to mira0435601-397 2014-07-01 16:59:50,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 3 to mira0437250-55 2014-07-01 16:59:50,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 4 to mira0437250-55 2014-07-01 16:59:50,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 5 to mira0435601-397 2014-07-01 16:59:50,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 158 (ObjNum 1828 snap 0 seq_num 1828) dirty exists 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: finishing write tid 7 to mira0437250-337 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 460 (ObjNum 834 snap 137 seq_num 834) dirty exists 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: done (11 left) 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2944: done (10 left) 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 145 current snap is 0 2014-07-01 16:59:50,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: seq_num 1845 ranges {3964928=737280} 2014-07-01 16:59:50,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 6 to mira0437250-55 2014-07-01 16:59:50,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: finishing write tid 7 to mira0437250-55 2014-07-01 16:59:50,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 735 (ObjNum 833 snap 137 seq_num 833) dirty exists 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: writing mira0435601-145 from 3964928 to 4702208 tid 1 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 261 current snap is 0 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: seq_num 1846 ranges {4698112=765952,5464064=733184,6197248=471040} 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: writing mira0435601-261 from 4698112 to 5464064 tid 1 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: writing mira0435601-261 from 5464064 to 6197248 tid 2 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: writing mira0435601-261 from 6197248 to 6668288 tid 3 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2947: read oid 422 snap -1 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: finishing write tid 1 to mira0435601-274 2014-07-01 16:59:51,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 16:59:51,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:1383: done (6 left) 2014-07-01 16:59:51,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1397: done (5 left) 2014-07-01 16:59:51,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1398: done (4 left) 2014-07-01 16:59:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1399: done (3 left) 2014-07-01 16:59:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1403: done (2 left) 2014-07-01 16:59:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1404: done (1 left) 2014-07-01 16:59:51,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: read oid 320 snap -1 2014-07-01 16:59:51,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1406: read oid 406 snap -1 2014-07-01 16:59:51,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: read oid 448 snap -1 2014-07-01 16:59:51,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1408: rollback oid 47 current snap is 138 2014-07-01 16:59:51,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 47 to 114 2014-07-01 16:59:51,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: write oid 64 current snap is 138 2014-07-01 16:59:51,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: seq_num 836 ranges {639904=576814,1439903=1} 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: writing mira0437250-64 from 639904 to 1216718 tid 1 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: writing mira0437250-64 from 1439903 to 1439904 tid 2 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: write oid 472 current snap is 138 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: seq_num 837 ranges {735134=400000} 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: writing mira0437250-472 from 735134 to 1135134 tid 1 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1411: delete oid 400 current snap is 138 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: expect (ObjNum 704 snap 93 seq_num 704) 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 612 (ObjNum 611 snap 47 seq_num 611) dirty exists 2014-07-01 16:59:51,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: expect (ObjNum 821 snap 130 seq_num 821) 2014-07-01 16:59:51,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: finishing write tid 1 to mira0435601-237 2014-07-01 16:59:51,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: finishing write tid 3 to mira0435601-237 2014-07-01 16:59:51,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: finishing write tid 3 to mira0435601-274 2014-07-01 16:59:51,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2942: expect (ObjNum 183 snap 0 seq_num 183) 2014-07-01 16:59:52,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:1406: expect (ObjNum 672 snap 78 seq_num 672) 2014-07-01 16:59:52,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: finishing write tid 4 to mira0435601-237 2014-07-01 16:59:52,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 120 (ObjNum 1841 snap 0 seq_num 1841) dirty exists 2014-07-01 16:59:52,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: expect (ObjNum 649 snap 0 seq_num 649) 2014-07-01 16:59:52,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: expect (ObjNum 319 snap 0 seq_num 319) 2014-07-01 16:59:53,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 1 to mira0437250-472 2014-07-01 16:59:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 3 to mira0437250-472 2014-07-01 16:59:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: finishing write tid 4 to mira0437250-472 2014-07-01 16:59:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 277 (ObjNum 837 snap 138 seq_num 837) dirty exists 2014-07-01 16:59:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1402: done (7 left) 2014-07-01 16:59:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1405: done (6 left) 2014-07-01 16:59:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1406: done (5 left) 2014-07-01 16:59:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1407: done (4 left) 2014-07-01 16:59:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1408: done (3 left) 2014-07-01 16:59:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1410: done (2 left) 2014-07-01 16:59:53,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1411: done (1 left) 2014-07-01 16:59:53,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:1412: delete oid 239 current snap is 138 2014-07-01 16:59:53,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1412: done (1 left) 2014-07-01 16:59:53,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: write oid 285 current snap is 138 2014-07-01 16:59:53,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: seq_num 840 ranges {436375=500127,1625746=798754,2836374=1} 2014-07-01 16:59:53,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: writing mira0437250-285 from 436375 to 936502 tid 1 2014-07-01 16:59:53,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: writing mira0437250-285 from 1625746 to 2424500 tid 2 2014-07-01 16:59:53,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: writing mira0437250-285 from 2836374 to 2836375 tid 3 2014-07-01 16:59:53,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: write oid 123 current snap is 138 2014-07-01 16:59:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: seq_num 841 ranges {531605=623356,1855207=676398} 2014-07-01 16:59:53,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: writing mira0437250-123 from 531605 to 1154961 tid 1 2014-07-01 16:59:54,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: writing mira0437250-123 from 1855207 to 2531605 tid 2 2014-07-01 16:59:54,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:1415: snap_create 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: finishing write tid 1 to mira0437250-285 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 1 to mira0437250-64 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 2 to mira0437250-64 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: finishing write tid 2 to mira0437250-285 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 4 to mira0437250-64 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: finishing write tid 3 to mira0437250-285 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: finishing write tid 5 to mira0437250-285 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: finishing write tid 5 to mira0437250-64 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 347 (ObjNum 836 snap 138 seq_num 836) dirty exists 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: finishing write tid 6 to mira0437250-285 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 506 (ObjNum 840 snap 138 seq_num 840) dirty exists 2014-07-01 16:59:54,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: finishing write tid 1 to mira0437250-123 2014-07-01 16:59:54,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 1 to mira0435601-368 2014-07-01 16:59:54,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 2 to mira0435601-368 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 3 to mira0435601-368 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 4 to mira0435601-368 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: finishing write tid 1 to mira0435601-95 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: finishing write tid 2 to mira0435601-95 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: finishing write tid 3 to mira0435601-95 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: finishing write tid 5 to mira0435601-95 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 6 to mira0435601-368 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: done (12 left) 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: done (11 left) 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: done (10 left) 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2942: done (9 left) 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: read oid 482 snap -1 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: finishing write tid 7 to mira0435601-368 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 155 (ObjNum 1835 snap 0 seq_num 1835) dirty exists 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: finishing write tid 6 to mira0435601-95 2014-07-01 16:59:54,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 280 (ObjNum 1840 snap 0 seq_num 1840) dirty exists 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: finishing write tid 1 to mira0435601-145 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: finishing write tid 3 to mira0435601-145 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: done (9 left) 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: done (8 left) 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 185 current snap is 0 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: seq_num 1847 ranges {61440=483328} 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: writing mira0435601-185 from 61440 to 544768 tid 1 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: read oid 298 snap -1 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: finishing write tid 4 to mira0435601-145 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 124 (ObjNum 1845 snap 0 seq_num 1845) dirty exists 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: finishing write tid 1 to mira0435601-261 2014-07-01 16:59:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: expect (ObjNum 736 snap 0 seq_num 736) 2014-07-01 16:59:54,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: finishing write tid 2 to mira0437250-123 2014-07-01 16:59:54,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: finishing write tid 4 to mira0437250-123 2014-07-01 16:59:54,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: finishing write tid 5 to mira0437250-123 2014-07-01 16:59:54,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 351 (ObjNum 841 snap 138 seq_num 841) dirty exists 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1409: done (3 left) 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1413: done (2 left) 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1414: done (1 left) 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1415: done (0 left) 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: rollback oid 339 current snap is 139 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 339 to 67 2014-07-01 16:59:54,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: write oid 129 current snap is 139 2014-07-01 16:59:54,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: seq_num 842 ranges {323985=1} 2014-07-01 16:59:54,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: writing mira0437250-129 from 323985 to 323986 tid 1 2014-07-01 16:59:54,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1418: rollback oid 242 current snap is 139 2014-07-01 16:59:54,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 242 to 34 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1419: rollback oid 481 current snap is 139 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 481 to 127 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: read oid 52 snap 80 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1421: snap_create 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 1 to mira0437250-129 2014-07-01 16:59:55,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: expect (ObjNum 51 snap 0 seq_num 51) 2014-07-01 16:59:55,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 3 to mira0437250-129 2014-07-01 16:59:55,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 412 (ObjNum 693 snap 88 seq_num 693) dirty exists 2014-07-01 16:59:55,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: finishing write tid 4 to mira0437250-129 2014-07-01 16:59:55,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 591 (ObjNum 842 snap 139 seq_num 842) dirty exists 2014-07-01 16:59:55,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 256 (ObjNum 338 snap 0 seq_num 338) dirty exists 2014-07-01 16:59:55,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 292 (ObjNum 241 snap 0 seq_num 241) dirty exists 2014-07-01 16:59:55,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: finishing write tid 1 to mira0435601-181 2014-07-01 16:59:55,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: finishing write tid 3 to mira0435601-181 2014-07-01 16:59:55,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: finishing write tid 4 to mira0435601-181 2014-07-01 16:59:55,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 356 (ObjNum 1843 snap 0 seq_num 1843) dirty exists 2014-07-01 16:59:55,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2947: expect (ObjNum 1659 snap 0 seq_num 1659) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1416: done (5 left) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1417: done (4 left) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1418: done (3 left) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1419: done (2 left) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1420: done (1 left) 2014-07-01 16:59:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1421: done (0 left) 2014-07-01 16:59:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:1422: read oid 347 snap -1 2014-07-01 16:59:55,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: write oid 355 current snap is 140 2014-07-01 16:59:55,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: seq_num 843 ranges {536591=1} 2014-07-01 16:59:55,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: writing mira0437250-355 from 536591 to 536592 tid 1 2014-07-01 16:59:55,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: read oid 359 snap 126 2014-07-01 16:59:56,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:1425: snap_create 2014-07-01 16:59:56,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: finishing write tid 1 to mira0437250-355 2014-07-01 16:59:56,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: finishing write tid 3 to mira0437250-355 2014-07-01 16:59:56,085.085 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 16:59:56,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: finishing write tid 4 to mira0437250-355 2014-07-01 16:59:56,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 259 (ObjNum 843 snap 140 seq_num 843) dirty exists 2014-07-01 16:59:56,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:1422: expect (ObjNum 677 snap 79 seq_num 677) 2014-07-01 16:59:56,851.851 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 16:59:57,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: finishing write tid 2 to mira0435601-261 2014-07-01 16:59:57,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: finishing write tid 3 to mira0435601-261 2014-07-01 16:59:57,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: finishing write tid 5 to mira0435601-261 2014-07-01 16:59:57,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: finishing write tid 6 to mira0435601-261 2014-07-01 16:59:57,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 308 (ObjNum 1846 snap 0 seq_num 1846) dirty exists 2014-07-01 16:59:57,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: expect (ObjNum 1592 snap 0 seq_num 1592) 2014-07-01 16:59:57,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: expect (ObjNum 358 snap 0 seq_num 358) 2014-07-01 16:59:59,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:1422: done (3 left) 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1423: done (2 left) 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1424: done (1 left) 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1425: done (0 left) 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1426: rollback oid 331 current snap is 141 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 331 to 90 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: write oid 161 current snap is 141 2014-07-01 16:59:59,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: seq_num 844 ranges {481422=574865,1281421=1} 2014-07-01 16:59:59,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: writing mira0437250-161 from 481422 to 1056287 tid 1 2014-07-01 16:59:59,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: writing mira0437250-161 from 1281421 to 1281422 tid 2 2014-07-01 16:59:59,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: rollback oid 162 current snap is 141 2014-07-01 16:59:59,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 162 to 91 2014-07-01 16:59:59,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 456 (ObjNum 330 snap 0 seq_num 330) dirty exists 2014-07-01 16:59:59,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:1426: done (2 left) 2014-07-01 16:59:59,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: write oid 277 current snap is 141 2014-07-01 16:59:59,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: seq_num 845 ranges {425228=699117,1573815=451413} 2014-07-01 16:59:59,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: writing mira0437250-277 from 425228 to 1124345 tid 1 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: writing mira0437250-277 from 1573815 to 2025228 tid 2 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: write oid 423 current snap is 141 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: seq_num 846 ranges {521481=554569,1687946=33535} 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: writing mira0437250-423 from 521481 to 1076050 tid 1 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: writing mira0437250-423 from 1687946 to 1721481 tid 2 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: read oid 166 snap -1 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 616 (ObjNum 161 snap 0 seq_num 161) dirty exists 2014-07-01 16:59:59,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: finishing write tid 1 to mira0437250-161 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: finishing write tid 2 to mira0437250-161 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: finishing write tid 4 to mira0437250-161 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: finishing write tid 5 to mira0437250-161 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 739 (ObjNum 844 snap 141 seq_num 844) dirty exists 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: finishing write tid 1 to mira0437250-277 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: finishing write tid 2 to mira0437250-277 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: finishing write tid 4 to mira0437250-277 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1427: done (4 left) 2014-07-01 16:59:59,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:1428: done (3 left) 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1432: rollback oid 435 current snap is 141 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 435 to 113 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: finishing write tid 5 to mira0437250-277 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 392 (ObjNum 845 snap 141 seq_num 845) dirty exists 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1429: done (3 left) 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1433: delete oid 64 current snap is 141 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 593 (ObjNum 700 snap 92 seq_num 700) dirty exists 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1432: done (3 left) 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1433: done (2 left) 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1434: snap_remove snap 139 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: finishing write tid 1 to mira0437250-423 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: finishing write tid 2 to mira0437250-423 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: finishing write tid 4 to mira0437250-423 2014-07-01 16:59:59,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: expect (ObjNum 681 snap 79 seq_num 681) 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: finishing write tid 5 to mira0437250-423 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 502 (ObjNum 846 snap 141 seq_num 846) dirty exists 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1430: done (2 left) 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1431: done (1 left) 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1434: done (0 left) 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: rollback oid 151 current snap is 141 2014-07-01 16:59:59,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 151 to 51 2014-07-01 16:59:59,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1436: snap_remove snap 87 2014-07-01 16:59:59,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 756 (ObjNum 150 snap 0 seq_num 150) dirty exists 2014-07-01 16:59:59,722.722 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:00:00,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1435: done (1 left) 2014-07-01 17:00:00,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1436: done (0 left) 2014-07-01 17:00:00,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: rollback oid 173 current snap is 141 2014-07-01 17:00:00,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 173 to 134 2014-07-01 17:00:00,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: delete oid 428 current snap is 141 2014-07-01 17:00:00,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 743 (ObjNum 172 snap 0 seq_num 172) dirty exists 2014-07-01 17:00:00,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1437: done (1 left) 2014-07-01 17:00:00,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1438: done (0 left) 2014-07-01 17:00:00,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: write oid 49 current snap is 141 2014-07-01 17:00:00,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: seq_num 849 ranges {621698=623454,1807750=568431,2832038=589660} 2014-07-01 17:00:00,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: writing mira0437250-49 from 621698 to 1245152 tid 1 2014-07-01 17:00:00,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: writing mira0437250-49 from 1807750 to 2376181 tid 2 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: writing mira0437250-49 from 2832038 to 3421698 tid 3 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: write oid 365 current snap is 141 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: seq_num 850 ranges {165503=1} 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: writing mira0437250-365 from 165503 to 165504 tid 1 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: write oid 12 current snap is 141 2014-07-01 17:00:00,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: seq_num 851 ranges {661758=602135,1883567=708821,3216313=645445} 2014-07-01 17:00:00,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: writing mira0437250-12 from 661758 to 1263893 tid 1 2014-07-01 17:00:00,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: writing mira0437250-12 from 1883567 to 2592388 tid 2 2014-07-01 17:00:00,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: writing mira0437250-12 from 3216313 to 3861758 tid 3 2014-07-01 17:00:00,468.468 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:59:44.725350', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881002872, u'health': u'HEALTH_OK', u'kb_used': 15778180, u'store_stats': {u'bytes_total': 5869066, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4820474}}, {u'last_updated': u'2014-07-01 07:59:54.543144', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885210588, u'health': u'HEALTH_OK', u'kb_used': 11570464, u'store_stats': {u'bytes_total': 6916736, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4819568}}, {u'last_updated': u'2014-07-01 07:59:11.325340', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885315256, u'health': u'HEALTH_OK', u'kb_used': 11465796, u'store_stats': {u'bytes_total': 17051989, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4469061}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'1 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 26/3234 objects degraded (0.804%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 26, u'num_pgs': 444, u'data_bytes': 2588408487, u'degraded_total': 3234, u'bytes_used': 6330568704, u'op_per_sec': 14, u'write_bytes_sec': 568773, u'version': 2150, u'pgs_by_state': [{u'count': 443, u'state_name': u'active+clean'}, {u'count': 1, u'state_name': u'active+recovering'}], u'read_bytes_sec': 1129479, u'degraded_ratio': u'0.804', u'bytes_avail': 5991968477184}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 920, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:00:00,468.468 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1442: delete oid 431 current snap is 141 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: finishing write tid 1 to mira0437250-49 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: finishing write tid 2 to mira0437250-49 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: finishing write tid 1 to mira0437250-365 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: finishing write tid 3 to mira0437250-365 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: finishing write tid 3 to mira0437250-49 2014-07-01 17:00:00,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: finishing write tid 5 to mira0437250-49 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: finishing write tid 6 to mira0437250-49 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 49 v 513 (ObjNum 849 snap 141 seq_num 849) dirty exists 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: finishing write tid 4 to mira0437250-365 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 365 v 523 (ObjNum 850 snap 141 seq_num 850) dirty exists 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: finishing write tid 1 to mira0437250-12 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: finishing write tid 2 to mira0437250-12 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1439: done (3 left) 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1440: done (2 left) 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1442: done (1 left) 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1443: snap_remove snap 137 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: finishing write tid 3 to mira0437250-12 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: finishing write tid 5 to mira0437250-12 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: finishing write tid 6 to mira0437250-12 2014-07-01 17:00:00,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 295 (ObjNum 851 snap 141 seq_num 851) dirty exists 2014-07-01 17:00:01,240.240 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 17:00:01,241.241 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:01,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:1441: done (1 left) 2014-07-01 17:00:01,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1443: done (0 left) 2014-07-01 17:00:01,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:1444: delete oid 381 current snap is 141 2014-07-01 17:00:01,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1444: done (0 left) 2014-07-01 17:00:01,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:1445: rollback oid 296 current snap is 141 2014-07-01 17:00:01,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 296 to 125 2014-07-01 17:00:01,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: read oid 494 snap 126 2014-07-01 17:00:01,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: delete oid 100 current snap is 141 2014-07-01 17:00:01,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1447: done (2 left) 2014-07-01 17:00:01,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1448: delete oid 66 current snap is 141 2014-07-01 17:00:01,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: expect (ObjNum 493 snap 0 seq_num 493) 2014-07-01 17:00:01,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 420 (ObjNum 295 snap 0 seq_num 295) dirty exists 2014-07-01 17:00:01,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1445: done (2 left) 2014-07-01 17:00:01,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1446: done (1 left) 2014-07-01 17:00:01,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1448: done (0 left) 2014-07-01 17:00:01,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: snap_remove snap 53 2014-07-01 17:00:02,012.012 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:02,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1449: done (0 left) 2014-07-01 17:00:02,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: write oid 214 current snap is 141 2014-07-01 17:00:02,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: seq_num 856 ranges {651635=401125,1583061=445269,2621418=430217} 2014-07-01 17:00:02,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: writing mira0437250-214 from 651635 to 1052760 tid 1 2014-07-01 17:00:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: writing mira0437250-214 from 1583061 to 2028330 tid 2 2014-07-01 17:00:02,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: writing mira0437250-214 from 2621418 to 3051635 tid 3 2014-07-01 17:00:02,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: read oid 137 snap 136 2014-07-01 17:00:02,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: read oid 179 snap -1 2014-07-01 17:00:02,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1453: read oid 95 snap -1 2014-07-01 17:00:02,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: snap_create 2014-07-01 17:00:02,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: finishing write tid 1 to mira0437250-214 2014-07-01 17:00:02,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: finishing write tid 2 to mira0437250-214 2014-07-01 17:00:02,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: finishing write tid 3 to mira0437250-214 2014-07-01 17:00:02,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: finishing write tid 5 to mira0437250-214 2014-07-01 17:00:02,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: finishing write tid 6 to mira0437250-214 2014-07-01 17:00:02,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 423 (ObjNum 856 snap 141 seq_num 856) dirty exists 2014-07-01 17:00:02,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: expect (ObjNum 651 snap 71 seq_num 651) 2014-07-01 17:00:03,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: finishing write tid 4 to mira0435601-274 2014-07-01 17:00:03,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 194 (ObjNum 1831 snap 0 seq_num 1831) dirty exists 2014-07-01 17:00:03,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: expect (ObjNum 1100 snap 0 seq_num 1100) 2014-07-01 17:00:03,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1453: expect (ObjNum 94 snap 0 seq_num 94) 2014-07-01 17:00:03,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: expect (ObjNum 1812 snap 0 seq_num 1812) 2014-07-01 17:00:03,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1450: done (4 left) 2014-07-01 17:00:03,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:1451: done (3 left) 2014-07-01 17:00:03,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1452: done (2 left) 2014-07-01 17:00:03,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1453: done (1 left) 2014-07-01 17:00:03,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1454: done (0 left) 2014-07-01 17:00:03,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: snap_create 2014-07-01 17:00:04,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1455: done (0 left) 2014-07-01 17:00:04,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: write oid 311 current snap is 143 2014-07-01 17:00:04,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: seq_num 857 ranges {463216=772929,1661096=666889,3070195=193021} 2014-07-01 17:00:04,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0437250-311 from 463216 to 1236145 tid 1 2014-07-01 17:00:04,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0437250-311 from 1661096 to 2327985 tid 2 2014-07-01 17:00:04,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: writing mira0437250-311 from 3070195 to 3263216 tid 3 2014-07-01 17:00:04,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1457: read oid 29 snap 71 2014-07-01 17:00:04,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 1 to mira0437250-311 2014-07-01 17:00:04,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1458: snap_create 2014-07-01 17:00:04,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 2 to mira0437250-311 2014-07-01 17:00:04,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 3 to mira0437250-311 2014-07-01 17:00:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 5 to mira0437250-311 2014-07-01 17:00:04,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: finishing write tid 6 to mira0437250-311 2014-07-01 17:00:04,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 619 (ObjNum 857 snap 143 seq_num 857) dirty exists 2014-07-01 17:00:05,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1456: done (2 left) 2014-07-01 17:00:05,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1457: done (1 left) 2014-07-01 17:00:05,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:1458: done (0 left) 2014-07-01 17:00:05,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: write oid 472 current snap is 144 2014-07-01 17:00:05,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: seq_num 858 ranges {7021=1} 2014-07-01 17:00:05,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: writing mira0437250-472 from 7021 to 7022 tid 1 2014-07-01 17:00:05,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: rollback oid 181 current snap is 144 2014-07-01 17:00:05,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 181 to 108 2014-07-01 17:00:05,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: snap_create 2014-07-01 17:00:05,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 509 (ObjNum 180 snap 0 seq_num 180) dirty exists 2014-07-01 17:00:05,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: finishing write tid 1 to mira0437250-472 2014-07-01 17:00:05,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: finishing write tid 3 to mira0437250-472 2014-07-01 17:00:05,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: finishing write tid 4 to mira0437250-472 2014-07-01 17:00:05,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 301 (ObjNum 858 snap 144 seq_num 858) dirty exists 2014-07-01 17:00:06,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1459: done (2 left) 2014-07-01 17:00:06,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1460: done (1 left) 2014-07-01 17:00:06,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1461: done (0 left) 2014-07-01 17:00:06,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: write oid 289 current snap is 145 2014-07-01 17:00:06,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: seq_num 859 ranges {751851=1} 2014-07-01 17:00:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: writing mira0437250-289 from 751851 to 751852 tid 1 2014-07-01 17:00:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: write oid 400 current snap is 145 2014-07-01 17:00:06,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: seq_num 860 ranges {447081=1} 2014-07-01 17:00:06,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: writing mira0437250-400 from 447081 to 447082 tid 1 2014-07-01 17:00:06,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:1464: snap_remove snap 111 2014-07-01 17:00:06,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 1 to mira0437250-289 2014-07-01 17:00:06,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 3 to mira0437250-289 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: finishing write tid 4 to mira0437250-289 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 319 (ObjNum 859 snap 145 seq_num 859) dirty exists 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 1 to mira0437250-400 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 3 to mira0437250-400 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: finishing write tid 4 to mira0437250-400 2014-07-01 17:00:06,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 371 (ObjNum 860 snap 145 seq_num 860) dirty exists 2014-07-01 17:00:06,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 1 to mira0435601-185 2014-07-01 17:00:06,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 3 to mira0435601-185 2014-07-01 17:00:06,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 4 to mira0435601-185 2014-07-01 17:00:06,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 153 (ObjNum 1847 snap 0 seq_num 1847) dirty exists 2014-07-01 17:00:06,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: done (9 left) 2014-07-01 17:00:06,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: done (8 left) 2014-07-01 17:00:06,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: done (7 left) 2014-07-01 17:00:06,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: done (6 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: done (5 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: done (4 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2947: done (3 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: done (2 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: done (1 left) 2014-07-01 17:00:06,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: done (0 left) 2014-07-01 17:00:06,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 371 current snap is 0 2014-07-01 17:00:06,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: seq_num 1848 ranges {0=507904,507904=499712,1007616=585728,1593344=118784} 2014-07-01 17:00:06,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: writing mira0435601-371 from 0 to 507904 tid 1 2014-07-01 17:00:06,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: writing mira0435601-371 from 507904 to 1007616 tid 2 2014-07-01 17:00:06,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: writing mira0435601-371 from 1007616 to 1593344 tid 3 2014-07-01 17:00:06,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: writing mira0435601-371 from 1593344 to 1712128 tid 4 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 326 current snap is 0 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: seq_num 1849 ranges {0=585728,585728=483328,1069056=319488} 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: writing mira0435601-326 from 0 to 585728 tid 1 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: writing mira0435601-326 from 585728 to 1069056 tid 2 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: writing mira0435601-326 from 1069056 to 1388544 tid 3 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: read oid 203 snap -1 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 1 to mira0435601-371 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 2 to mira0435601-371 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 3 to mira0435601-371 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 4 to mira0435601-371 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 6 to mira0435601-371 2014-07-01 17:00:07,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: finishing write tid 7 to mira0435601-371 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 355 (ObjNum 1848 snap 0 seq_num 1848) dirty exists 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: finishing write tid 1 to mira0435601-326 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: done (2 left) 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2954: delete oid 273 current snap is 0 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: done (2 left) 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2954: done (1 left) 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2955: delete oid 306 current snap is 0 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: finishing write tid 2 to mira0435601-326 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2955: done (1 left) 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 17:00:07,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: seq_num 1852 ranges {5652480=757760,6410240=479232,6889472=634880,7524352=491520} 2014-07-01 17:00:07,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: writing mira0435601-288 from 5652480 to 6410240 tid 1 2014-07-01 17:00:07,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: writing mira0435601-288 from 6410240 to 6889472 tid 2 2014-07-01 17:00:07,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: writing mira0435601-288 from 6889472 to 7524352 tid 3 2014-07-01 17:00:07,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: writing mira0435601-288 from 7524352 to 8015872 tid 4 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: delete oid 352 current snap is 0 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: finishing write tid 3 to mira0435601-326 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: finishing write tid 5 to mira0435601-326 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: finishing write tid 6 to mira0435601-326 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 126 (ObjNum 1849 snap 0 seq_num 1849) dirty exists 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 1 to mira0435601-288 2014-07-01 17:00:07,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 2 to mira0435601-288 2014-07-01 17:00:07,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 3 to mira0435601-288 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 4 to mira0435601-288 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: done (2 left) 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: done (1 left) 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: read oid 147 snap -1 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 457 current snap is 0 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: seq_num 1854 ranges {3424256=548864} 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: writing mira0435601-457 from 3424256 to 3973120 tid 1 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2960: read oid 312 snap -1 2014-07-01 17:00:07,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 6 to mira0435601-288 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 7 to mira0435601-288 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 360 (ObjNum 1852 snap 0 seq_num 1852) dirty exists 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: done (3 left) 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 281 current snap is 0 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: seq_num 1855 ranges {0=577536,577536=475136,1052672=405504,1458176=323584} 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: writing mira0435601-281 from 0 to 577536 tid 1 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: writing mira0435601-281 from 577536 to 1052672 tid 2 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: writing mira0435601-281 from 1052672 to 1458176 tid 3 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: writing mira0435601-281 from 1458176 to 1781760 tid 4 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2962: read oid 131 snap -1 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: finishing write tid 1 to mira0435601-457 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: finishing write tid 3 to mira0435601-457 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: finishing write tid 4 to mira0435601-457 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 225 (ObjNum 1854 snap 0 seq_num 1854) dirty exists 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: done (4 left) 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 123 current snap is 0 2014-07-01 17:00:07,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: seq_num 1856 ranges {2076672=446464,2523136=704512,3227648=499712} 2014-07-01 17:00:07,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: writing mira0435601-123 from 2076672 to 2523136 tid 1 2014-07-01 17:00:07,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: writing mira0435601-123 from 2523136 to 3227648 tid 2 2014-07-01 17:00:07,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: writing mira0435601-123 from 3227648 to 3727360 tid 3 2014-07-01 17:00:07,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 381 current snap is 0 2014-07-01 17:00:07,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: seq_num 1857 ranges {1982464=720896,2703360=688128,3391488=114688} 2014-07-01 17:00:07,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1462: done (2 left) 2014-07-01 17:00:07,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1463: done (1 left) 2014-07-01 17:00:07,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1464: done (0 left) 2014-07-01 17:00:07,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: delete oid 212 current snap is 145 2014-07-01 17:00:07,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: writing mira0435601-381 from 1982464 to 2703360 tid 1 2014-07-01 17:00:07,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: writing mira0435601-381 from 2703360 to 3391488 tid 2 2014-07-01 17:00:07,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: writing mira0435601-381 from 3391488 to 3506176 tid 3 2014-07-01 17:00:07,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2965: read oid 81 snap -1 2014-07-01 17:00:07,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: finishing write tid 1 to mira0435601-123 2014-07-01 17:00:07,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: expect (ObjNum 1796 snap 0 seq_num 1796) 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1465: done (0 left) 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: read oid 50 snap -1 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: write oid 203 current snap is 145 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: seq_num 862 ranges {453093=419471,1609165=769289,2453092=1} 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0437250-203 from 453093 to 872564 tid 1 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0437250-203 from 1609165 to 2378454 tid 2 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: writing mira0437250-203 from 2453092 to 2453093 tid 3 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: read oid 328 snap 90 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1469: rollback oid 140 current snap is 145 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 140 to 121 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: read oid 451 snap 138 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: write oid 106 current snap is 145 2014-07-01 17:00:07,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: seq_num 863 ranges {548323=675947,1854497=293826} 2014-07-01 17:00:07,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: writing mira0437250-106 from 548323 to 1224270 tid 1 2014-07-01 17:00:07,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: writing mira0437250-106 from 1854497 to 2148323 tid 2 2014-07-01 17:00:07,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: read oid 113 snap -1 2014-07-01 17:00:07,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: expect (ObjNum 49 snap 0 seq_num 49) 2014-07-01 17:00:07,860.860 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:08,622.622 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:09,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 140 v 482 (ObjNum 771 snap 116 seq_num 771) dirty exists 2014-07-01 17:00:09,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 1 to mira0437250-203 2014-07-01 17:00:09,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 2 to mira0437250-203 2014-07-01 17:00:09,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 3 to mira0437250-203 2014-07-01 17:00:09,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 5 to mira0437250-203 2014-07-01 17:00:09,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: expect (ObjNum 327 snap 0 seq_num 327) 2014-07-01 17:00:10,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: finishing write tid 6 to mira0437250-203 2014-07-01 17:00:10,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 443 (ObjNum 862 snap 145 seq_num 862) dirty exists 2014-07-01 17:00:10,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: expect (ObjNum 719 snap 97 seq_num 719) 2014-07-01 17:00:11,526.526 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:00:11,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2962: expect (ObjNum 815 snap 0 seq_num 815) 2014-07-01 17:00:12,242.242 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 8, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.014817', u'skew': u'0.000978', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.007322', u'skew': u'0.000935', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:59:44.725350', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881002872, u'health': u'HEALTH_OK', u'kb_used': 15778180, u'store_stats': {u'bytes_total': 5869066, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4820474}}, {u'last_updated': u'2014-07-01 07:59:54.543144', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885210588, u'health': u'HEALTH_OK', u'kb_used': 11570464, u'store_stats': {u'bytes_total': 6916736, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4819568}}, {u'last_updated': u'2014-07-01 08:00:11.325798', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885178984, u'health': u'HEALTH_OK', u'kb_used': 11602068, u'store_stats': {u'bytes_total': 10062471, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4819575}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 444, u'data_bytes': 2612539560, u'bytes_used': 6109511680, u'op_per_sec': 33, u'version': 2164, u'pgs_by_state': [{u'count': 444, u'state_name': u'active+clean'}], u'read_bytes_sec': 10008950, u'write_bytes_sec': 5106089, u'bytes_avail': 5992189534208}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 927, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:00:12,242.242 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:00:12,242.242 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:00:12,242.242 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_1 2014-07-01 17:00:12,242.242 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: finishing write tid 1 to mira0437250-106 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: finishing write tid 2 to mira0437250-106 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: finishing write tid 4 to mira0437250-106 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: finishing write tid 5 to mira0437250-106 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 643 (ObjNum 863 snap 145 seq_num 863) dirty exists 2014-07-01 17:00:12,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1466: done (6 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1467: done (5 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1468: done (4 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1469: done (3 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1470: done (2 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1471: done (1 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1472: done (0 left) 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: write oid 500 current snap is 145 2014-07-01 17:00:12,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: seq_num 864 ranges {644577=799176,1844576=1} 2014-07-01 17:00:12,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: writing mira0437250-500 from 644577 to 1443753 tid 1 2014-07-01 17:00:12,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: writing mira0437250-500 from 1844576 to 1844577 tid 2 2014-07-01 17:00:12,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: write oid 410 current snap is 145 2014-07-01 17:00:12,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: seq_num 865 ranges {436958=770980,1646665=614946,2757949=527174,3636957=1} 2014-07-01 17:00:12,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0437250-410 from 436958 to 1207938 tid 1 2014-07-01 17:00:12,989.989 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:13,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0437250-410 from 1646665 to 2261611 tid 2 2014-07-01 17:00:13,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0437250-410 from 2757949 to 3285123 tid 3 2014-07-01 17:00:13,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: writing mira0437250-410 from 3636957 to 3636958 tid 4 2014-07-01 17:00:13,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: read oid 477 snap -1 2014-07-01 17:00:13,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: finishing write tid 1 to mira0437250-500 2014-07-01 17:00:13,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: finishing write tid 2 to mira0437250-500 2014-07-01 17:00:13,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: finishing write tid 4 to mira0437250-500 2014-07-01 17:00:13,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: finishing write tid 5 to mira0437250-500 2014-07-01 17:00:13,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 500 v 370 (ObjNum 864 snap 145 seq_num 864) dirty exists 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 1 to mira0437250-410 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 2 to mira0437250-410 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1473: done (2 left) 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: read oid 252 snap 136 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: read oid 226 snap -1 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1478: delete oid 341 current snap is 145 2014-07-01 17:00:13,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1478: done (4 left) 2014-07-01 17:00:13,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1479: snap_remove snap 67 2014-07-01 17:00:13,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: expect (ObjNum 225 snap 0 seq_num 225) 2014-07-01 17:00:13,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: expect (ObjNum 680 snap 79 seq_num 680) 2014-07-01 17:00:13,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2960: expect (ObjNum 1712 snap 0 seq_num 1712) 2014-07-01 17:00:13,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 3 to mira0437250-410 2014-07-01 17:00:13,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 4 to mira0437250-410 2014-07-01 17:00:13,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 6 to mira0437250-410 2014-07-01 17:00:13,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: expect (ObjNum 666 snap 75 seq_num 666) 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 1 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 2 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 3 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 4 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 6 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: finishing write tid 7 to mira0435601-281 2014-07-01 17:00:13,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 161 (ObjNum 1855 snap 0 seq_num 1855) dirty exists 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: finishing write tid 2 to mira0435601-123 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: finishing write tid 3 to mira0435601-123 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: finishing write tid 5 to mira0435601-123 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: done (6 left) 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2960: done (5 left) 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: done (4 left) 2014-07-01 17:00:13,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:2962: done (3 left) 2014-07-01 17:00:13,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 370 current snap is 0 2014-07-01 17:00:13,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: seq_num 1858 ranges {884736=798720,1683456=401408} 2014-07-01 17:00:13,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: writing mira0435601-370 from 884736 to 1683456 tid 1 2014-07-01 17:00:13,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: writing mira0435601-370 from 1683456 to 2084864 tid 2 2014-07-01 17:00:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2967: read oid 40 snap -1 2014-07-01 17:00:14,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: finishing write tid 1 to mira0435601-381 2014-07-01 17:00:14,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: finishing write tid 6 to mira0435601-123 2014-07-01 17:00:14,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 231 (ObjNum 1856 snap 0 seq_num 1856) dirty exists 2014-07-01 17:00:14,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2965: expect (ObjNum 80 snap 0 seq_num 80) 2014-07-01 17:00:14,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: finishing write tid 7 to mira0437250-410 2014-07-01 17:00:14,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 539 (ObjNum 865 snap 145 seq_num 865) dirty exists 2014-07-01 17:00:14,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1474: done (4 left) 2014-07-01 17:00:14,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1475: done (3 left) 2014-07-01 17:00:14,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1476: done (2 left) 2014-07-01 17:00:14,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:1477: done (1 left) 2014-07-01 17:00:14,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:1479: done (0 left) 2014-07-01 17:00:14,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: read oid 361 snap 79 2014-07-01 17:00:14,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: delete oid 405 current snap is 145 2014-07-01 17:00:14,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: expect (ObjNum 360 snap 0 seq_num 360) 2014-07-01 17:00:14,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: finishing write tid 2 to mira0435601-381 2014-07-01 17:00:14,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: finishing write tid 3 to mira0435601-381 2014-07-01 17:00:14,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: finishing write tid 5 to mira0435601-381 2014-07-01 17:00:14,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: finishing write tid 6 to mira0435601-381 2014-07-01 17:00:14,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 262 (ObjNum 1857 snap 0 seq_num 1857) dirty exists 2014-07-01 17:00:14,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2967: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 17:00:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: finishing write tid 1 to mira0435601-370 2014-07-01 17:00:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: finishing write tid 2 to mira0435601-370 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: finishing write tid 4 to mira0435601-370 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: finishing write tid 5 to mira0435601-370 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 159 (ObjNum 1858 snap 0 seq_num 1858) dirty exists 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: done (4 left) 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: done (3 left) 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2965: done (2 left) 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: done (1 left) 2014-07-01 17:00:14,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2967: done (0 left) 2014-07-01 17:00:14,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 236 current snap is 0 2014-07-01 17:00:14,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: seq_num 1859 ranges {1048576=667648,1716224=401408} 2014-07-01 17:00:14,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: writing mira0435601-236 from 1048576 to 1716224 tid 1 2014-07-01 17:00:14,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: writing mira0435601-236 from 1716224 to 2117632 tid 2 2014-07-01 17:00:14,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2969: read oid 261 snap -1 2014-07-01 17:00:14,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 17:00:14,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: seq_num 1860 ranges {9416704=540672,9957376=401408} 2014-07-01 17:00:15,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: writing mira0435601-97 from 9416704 to 9957376 tid 1 2014-07-01 17:00:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: writing mira0435601-97 from 9957376 to 10358784 tid 2 2014-07-01 17:00:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 98 current snap is 0 2014-07-01 17:00:15,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: seq_num 1861 ranges {5013504=413696,5427200=401408} 2014-07-01 17:00:15,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: writing mira0435601-98 from 5013504 to 5427200 tid 1 2014-07-01 17:00:15,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: writing mira0435601-98 from 5427200 to 5828608 tid 2 2014-07-01 17:00:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: read oid 193 snap -1 2014-07-01 17:00:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: finishing write tid 1 to mira0435601-236 2014-07-01 17:00:15,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: finishing write tid 2 to mira0435601-236 2014-07-01 17:00:15,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: finishing write tid 4 to mira0435601-236 2014-07-01 17:00:15,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: finishing write tid 5 to mira0435601-236 2014-07-01 17:00:15,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 164 (ObjNum 1859 snap 0 seq_num 1859) dirty exists 2014-07-01 17:00:15,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2969: expect (ObjNum 1846 snap 0 seq_num 1846) 2014-07-01 17:00:15,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1480: done (1 left) 2014-07-01 17:00:15,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1481: done (0 left) 2014-07-01 17:00:15,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: read oid 262 snap 134 2014-07-01 17:00:15,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: write oid 66 current snap is 145 2014-07-01 17:00:15,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: seq_num 868 ranges {537175=473913,1337174=1} 2014-07-01 17:00:15,891.891 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:00:15,892.892 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pg_num 86' 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: writing mira0437250-66 from 537175 to 1011088 tid 1 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: writing mira0437250-66 from 1337174 to 1337175 tid 2 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: delete oid 36 current snap is 145 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: expect (ObjNum 261 snap 0 seq_num 261) 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1482: done (2 left) 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1484: done (1 left) 2014-07-01 17:00:15,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: snap_remove snap 138 2014-07-01 17:00:15,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 1 to mira0437250-66 2014-07-01 17:00:16,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 2 to mira0437250-66 2014-07-01 17:00:16,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 4 to mira0437250-66 2014-07-01 17:00:16,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: finishing write tid 5 to mira0437250-66 2014-07-01 17:00:16,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 532 (ObjNum 868 snap 145 seq_num 868) dirty exists 2014-07-01 17:00:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1483: done (1 left) 2014-07-01 17:00:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1485: done (0 left) 2014-07-01 17:00:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: write oid 247 current snap is 145 2014-07-01 17:00:16,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: seq_num 870 ranges {728658=1} 2014-07-01 17:00:16,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: writing mira0437250-247 from 728658 to 728659 tid 1 2014-07-01 17:00:16,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: snap_create 2014-07-01 17:00:16,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: finishing write tid 1 to mira0437250-247 2014-07-01 17:00:16,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: finishing write tid 3 to mira0437250-247 2014-07-01 17:00:16,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: finishing write tid 4 to mira0437250-247 2014-07-01 17:00:16,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 516 (ObjNum 870 snap 145 seq_num 870) dirty exists 2014-07-01 17:00:17,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:1486: done (1 left) 2014-07-01 17:00:17,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:1487: done (0 left) 2014-07-01 17:00:17,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: delete oid 164 current snap is 146 2014-07-01 17:00:17,139.139 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pg_num to 86 2014-07-01 17:00:17,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1488: done (0 left) 2014-07-01 17:00:17,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: rollback oid 38 current snap is 146 2014-07-01 17:00:17,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 38 to 141 2014-07-01 17:00:17,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: snap_remove snap 114 2014-07-01 17:00:17,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 498 (ObjNum 37 snap 0 seq_num 37) dirty exists 2014-07-01 17:00:18,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1489: done (1 left) 2014-07-01 17:00:18,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:1490: done (0 left) 2014-07-01 17:00:18,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: read oid 96 snap -1 2014-07-01 17:00:18,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1492: snap_remove snap 144 2014-07-01 17:00:19,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1491: done (1 left) 2014-07-01 17:00:19,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1492: done (0 left) 2014-07-01 17:00:19,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: snap_remove snap 121 2014-07-01 17:00:19,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: finishing write tid 1 to mira0435601-97 2014-07-01 17:00:19,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: finishing write tid 2 to mira0435601-97 2014-07-01 17:00:19,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: finishing write tid 4 to mira0435601-97 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: finishing write tid 5 to mira0435601-97 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 311 (ObjNum 1860 snap 0 seq_num 1860) dirty exists 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 1 to mira0435601-98 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: done (4 left) 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2969: done (3 left) 2014-07-01 17:00:19,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: done (2 left) 2014-07-01 17:00:19,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 370 current snap is 0 2014-07-01 17:00:19,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: seq_num 1862 ranges {2084864=487424} 2014-07-01 17:00:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: writing mira0435601-370 from 2084864 to 2572288 tid 1 2014-07-01 17:00:19,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2974: read oid 425 snap -1 2014-07-01 17:00:19,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 2 to mira0435601-98 2014-07-01 17:00:19,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 4 to mira0435601-98 2014-07-01 17:00:19,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 5 to mira0435601-98 2014-07-01 17:00:19,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 275 (ObjNum 1861 snap 0 seq_num 1861) dirty exists 2014-07-01 17:00:19,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: expect (ObjNum 1750 snap 0 seq_num 1750) 2014-07-01 17:00:20,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:1493: done (0 left) 2014-07-01 17:00:20,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:1494: delete oid 367 current snap is 146 2014-07-01 17:00:20,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:1494: done (0 left) 2014-07-01 17:00:20,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: read oid 364 snap 118 2014-07-01 17:00:20,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: read oid 375 snap -1 2014-07-01 17:00:20,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: read oid 491 snap -1 2014-07-01 17:00:20,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: rollback oid 386 current snap is 146 2014-07-01 17:00:20,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 386 to 34 2014-07-01 17:00:20,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:1499: snap_remove snap 125 2014-07-01 17:00:20,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 392 (ObjNum 385 snap 0 seq_num 385) dirty exists 2014-07-01 17:00:20,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: expect (ObjNum 490 snap 0 seq_num 490) 2014-07-01 17:00:20,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: expect (ObjNum 538 snap 14 seq_num 538) 2014-07-01 17:00:21,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: expect (ObjNum 374 snap 0 seq_num 374) 2014-07-01 17:00:22,164.164 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 0, 5, 4, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:00:22,164.164 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 17:00:22,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1495: done (4 left) 2014-07-01 17:00:22,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1496: done (3 left) 2014-07-01 17:00:22,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1497: done (2 left) 2014-07-01 17:00:22,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1498: done (1 left) 2014-07-01 17:00:22,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1499: done (0 left) 2014-07-01 17:00:22,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: delete oid 282 current snap is 146 2014-07-01 17:00:22,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1500: done (0 left) 2014-07-01 17:00:22,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: write oid 305 current snap is 146 2014-07-01 17:00:22,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: seq_num 874 ranges {773706=493283,1730495=783599,3173705=1} 2014-07-01 17:00:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: writing mira0437250-305 from 773706 to 1266989 tid 1 2014-07-01 17:00:22,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: writing mira0437250-305 from 1730495 to 2514094 tid 2 2014-07-01 17:00:22,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: writing mira0437250-305 from 3173705 to 3173706 tid 3 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: snap_remove snap 90 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 1 to mira0437250-305 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 2 to mira0437250-305 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 3 to mira0437250-305 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 5 to mira0437250-305 2014-07-01 17:00:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: finishing write tid 6 to mira0437250-305 2014-07-01 17:00:22,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 564 (ObjNum 874 snap 146 seq_num 874) dirty exists 2014-07-01 17:00:23,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1501: done (1 left) 2014-07-01 17:00:23,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1502: done (0 left) 2014-07-01 17:00:23,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: write oid 184 current snap is 146 2014-07-01 17:00:23,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: seq_num 875 ranges {433863=1} 2014-07-01 17:00:23,224.224 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 17:00:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: writing mira0437250-184 from 433863 to 433864 tid 1 2014-07-01 17:00:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: read oid 145 snap 134 2014-07-01 17:00:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:1505: snap_create 2014-07-01 17:00:23,234.234 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 17:00:23,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: finishing write tid 1 to mira0437250-184 2014-07-01 17:00:23,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: expect (ObjNum 144 snap 0 seq_num 144) 2014-07-01 17:00:24,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 1 to mira0435601-370 2014-07-01 17:00:24,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 3 to mira0435601-370 2014-07-01 17:00:24,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 4 to mira0435601-370 2014-07-01 17:00:24,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 161 (ObjNum 1862 snap 0 seq_num 1862) dirty exists 2014-07-01 17:00:24,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:2974: expect (ObjNum 1146 snap 0 seq_num 1146) 2014-07-01 17:00:24,254.254 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 17:00:24,264.264 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 17:00:24,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: finishing write tid 3 to mira0437250-184 2014-07-01 17:00:24,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: finishing write tid 4 to mira0437250-184 2014-07-01 17:00:24,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 320 (ObjNum 875 snap 146 seq_num 875) dirty exists 2014-07-01 17:00:24,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1503: done (2 left) 2014-07-01 17:00:24,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:1504: done (1 left) 2014-07-01 17:00:24,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:1505: done (0 left) 2014-07-01 17:00:24,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:1506: read oid 374 snap -1 2014-07-01 17:00:24,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: write oid 292 current snap is 147 2014-07-01 17:00:24,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: seq_num 876 ranges {130116=1} 2014-07-01 17:00:24,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: writing mira0437250-292 from 130116 to 130117 tid 1 2014-07-01 17:00:24,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: read oid 420 snap -1 2014-07-01 17:00:24,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:1509: snap_create 2014-07-01 17:00:24,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 1 to mira0437250-292 2014-07-01 17:00:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 3 to mira0437250-292 2014-07-01 17:00:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: finishing write tid 4 to mira0437250-292 2014-07-01 17:00:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 902 (ObjNum 876 snap 147 seq_num 876) dirty exists 2014-07-01 17:00:24,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:1506: expect (ObjNum 373 snap 0 seq_num 373) 2014-07-01 17:00:25,276.276 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 17:00:25,291.291 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 17:00:25,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: expect (ObjNum 419 snap 0 seq_num 419) 2014-07-01 17:00:25,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1506: done (3 left) 2014-07-01 17:00:25,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1507: done (2 left) 2014-07-01 17:00:25,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1508: done (1 left) 2014-07-01 17:00:25,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1509: done (0 left) 2014-07-01 17:00:25,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:1510: delete oid 44 current snap is 148 2014-07-01 17:00:25,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1510: done (0 left) 2014-07-01 17:00:25,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:1511: snap_remove snap 108 2014-07-01 17:00:26,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: done (3 left) 2014-07-01 17:00:26,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: done (2 left) 2014-07-01 17:00:26,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: done (1 left) 2014-07-01 17:00:26,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:2974: done (0 left) 2014-07-01 17:00:26,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: read oid 324 snap -1 2014-07-01 17:00:26,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 454 current snap is 0 2014-07-01 17:00:26,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: seq_num 1863 ranges {0=516096,516096=434176,950272=700416,1650688=69632} 2014-07-01 17:00:26,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: writing mira0435601-454 from 0 to 516096 tid 1 2014-07-01 17:00:26,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1511: done (0 left) 2014-07-01 17:00:26,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:1512: rollback oid 147 current snap is 148 2014-07-01 17:00:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 147 to 142 2014-07-01 17:00:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: read oid 69 snap -1 2014-07-01 17:00:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: read oid 350 snap 34 2014-07-01 17:00:26,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: delete oid 184 current snap is 148 2014-07-01 17:00:26,256.256 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 17:00:26,269.269 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: writing mira0435601-454 from 516096 to 950272 tid 2 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: writing mira0435601-454 from 950272 to 1650688 tid 3 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: writing mira0435601-454 from 1650688 to 1720320 tid 4 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 178 current snap is 0 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: seq_num 1864 ranges {0=233472} 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: writing mira0435601-178 from 0 to 233472 tid 1 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 303 current snap is 0 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: seq_num 1865 ranges {0=659456,659456=495616,1155072=307200} 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: writing mira0435601-303 from 0 to 659456 tid 1 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: writing mira0435601-303 from 659456 to 1155072 tid 2 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: writing mira0435601-303 from 1155072 to 1462272 tid 3 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 495 current snap is 0 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: seq_num 1866 ranges {4534272=286720} 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1515: done (3 left) 2014-07-01 17:00:26,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: snap_remove snap 91 2014-07-01 17:00:26,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: expect (ObjNum 349 snap 0 seq_num 349) 2014-07-01 17:00:26,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: writing mira0435601-495 from 4534272 to 4820992 tid 1 2014-07-01 17:00:26,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: read oid 337 snap -1 2014-07-01 17:00:26,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 1 to mira0435601-454 2014-07-01 17:00:26,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 2 to mira0435601-454 2014-07-01 17:00:26,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 3 to mira0435601-454 2014-07-01 17:00:26,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: finishing write tid 1 to mira0435601-178 2014-07-01 17:00:26,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 4 to mira0435601-454 2014-07-01 17:00:26,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: done (5 left) 2014-07-01 17:00:26,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 144 current snap is 0 2014-07-01 17:00:26,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: seq_num 1867 ranges {49152=454656,503808=663552,1167360=749568,1916928=192512} 2014-07-01 17:00:26,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0435601-144 from 49152 to 503808 tid 1 2014-07-01 17:00:26,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0435601-144 from 503808 to 1167360 tid 2 2014-07-01 17:00:26,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0435601-144 from 1167360 to 1916928 tid 3 2014-07-01 17:00:26,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0435601-144 from 1916928 to 2109440 tid 4 2014-07-01 17:00:26,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 70 current snap is 0 2014-07-01 17:00:26,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: seq_num 1868 ranges {2125824=479232,2605056=401408} 2014-07-01 17:00:26,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: writing mira0435601-70 from 2125824 to 2605056 tid 1 2014-07-01 17:00:26,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: writing mira0435601-70 from 2605056 to 3006464 tid 2 2014-07-01 17:00:26,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 161 current snap is 0 2014-07-01 17:00:26,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: seq_num 1869 ranges {2469888=507904,2977792=475136,3452928=536576,3989504=462848,4452352=131072} 2014-07-01 17:00:26,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: writing mira0435601-161 from 2469888 to 2977792 tid 1 2014-07-01 17:00:26,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: writing mira0435601-161 from 2977792 to 3452928 tid 2 2014-07-01 17:00:26,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: writing mira0435601-161 from 3452928 to 3989504 tid 3 2014-07-01 17:00:26,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: writing mira0435601-161 from 3989504 to 4452352 tid 4 2014-07-01 17:00:26,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: writing mira0435601-161 from 4452352 to 4583424 tid 5 2014-07-01 17:00:26,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2984: delete oid 98 current snap is 0 2014-07-01 17:00:26,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: finishing write tid 3 to mira0435601-178 2014-07-01 17:00:26,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: finishing write tid 1 to mira0435601-303 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: finishing write tid 2 to mira0435601-303 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 6 to mira0435601-454 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: finishing write tid 3 to mira0435601-303 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: finishing write tid 5 to mira0435601-303 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 7 to mira0435601-454 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 169 (ObjNum 1863 snap 0 seq_num 1863) dirty exists 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: finishing write tid 4 to mira0435601-178 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 336 (ObjNum 1864 snap 0 seq_num 1864) dirty exists 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: finishing write tid 6 to mira0435601-303 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 212 (ObjNum 1865 snap 0 seq_num 1865) dirty exists 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: finishing write tid 1 to mira0435601-495 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: finishing write tid 3 to mira0435601-495 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: finishing write tid 4 to mira0435601-495 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 214 (ObjNum 1866 snap 0 seq_num 1866) dirty exists 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 1 to mira0435601-144 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 2 to mira0435601-144 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 3 to mira0435601-144 2014-07-01 17:00:26,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 4 to mira0435601-144 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 6 to mira0435601-144 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 7 to mira0435601-144 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 127 (ObjNum 1867 snap 0 seq_num 1867) dirty exists 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: finishing write tid 1 to mira0435601-70 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: finishing write tid 2 to mira0435601-70 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: finishing write tid 4 to mira0435601-70 2014-07-01 17:00:26,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: expect (ObjNum 1654 snap 0 seq_num 1654) 2014-07-01 17:00:27,271.271 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 17:00:27,287.287 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 17:00:27,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: expect (ObjNum 68 snap 0 seq_num 68) 2014-07-01 17:00:28,289.289 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 17:00:28,300.300 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 17:00:28,300.300 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:28,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 524 (ObjNum 570 snap 27 seq_num 570) dirty exists 2014-07-01 17:00:28,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1512: done (3 left) 2014-07-01 17:00:28,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:1513: done (2 left) 2014-07-01 17:00:28,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1514: done (1 left) 2014-07-01 17:00:28,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1516: done (0 left) 2014-07-01 17:00:28,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: read oid 195 snap -1 2014-07-01 17:00:28,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:1518: snap_remove snap 131 2014-07-01 17:00:28,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: expect (ObjNum 757 snap 111 seq_num 757) 2014-07-01 17:00:29,199.199 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:29,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:1517: done (1 left) 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1518: done (0 left) 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: rollback oid 222 current snap is 148 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 222 to 51 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: read oid 250 snap 136 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: read oid 98 snap -1 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:1522: rollback oid 497 current snap is 148 2014-07-01 17:00:29,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 497 to 112 2014-07-01 17:00:29,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:1523: rollback oid 273 current snap is 148 2014-07-01 17:00:29,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 273 to 127 2014-07-01 17:00:29,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: snap_remove snap 126 2014-07-01 17:00:29,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 0 (ObjNum 3 snap 0 seq_num 0) dirty dne 2014-07-01 17:00:29,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 0 (ObjNum 1 snap 1 seq_num 16342352) dirty dne 2014-07-01 17:00:29,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 612 (ObjNum 272 snap 0 seq_num 272) dirty exists 2014-07-01 17:00:29,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 17:00:31,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: expect (ObjNum 249 snap 0 seq_num 249) 2014-07-01 17:00:31,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1519: done (5 left) 2014-07-01 17:00:31,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:1520: done (4 left) 2014-07-01 17:00:31,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1521: done (3 left) 2014-07-01 17:00:31,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1522: done (2 left) 2014-07-01 17:00:31,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1523: done (1 left) 2014-07-01 17:00:31,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:1524: done (0 left) 2014-07-01 17:00:31,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: write oid 84 current snap is 148 2014-07-01 17:00:31,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: seq_num 879 ranges {515007=424497,1315006=1} 2014-07-01 17:00:31,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: writing mira0437250-84 from 515007 to 939504 tid 1 2014-07-01 17:00:31,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: writing mira0437250-84 from 1315006 to 1315007 tid 2 2014-07-01 17:00:31,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:1526: snap_remove snap 142 2014-07-01 17:00:31,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 1 to mira0437250-84 2014-07-01 17:00:31,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 2 to mira0437250-84 2014-07-01 17:00:31,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 4 to mira0437250-84 2014-07-01 17:00:31,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: finishing write tid 5 to mira0437250-84 2014-07-01 17:00:31,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 511 (ObjNum 879 snap 148 seq_num 879) dirty exists 2014-07-01 17:00:32,132.132 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:32,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:1525: done (1 left) 2014-07-01 17:00:32,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1526: done (0 left) 2014-07-01 17:00:32,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1527: delete oid 41 current snap is 148 2014-07-01 17:00:32,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1527: done (0 left) 2014-07-01 17:00:32,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:1528: snap_remove snap 122 2014-07-01 17:00:32,862.862 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:33,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1528: done (0 left) 2014-07-01 17:00:33,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:1529: rollback oid 74 current snap is 148 2014-07-01 17:00:33,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 74 to 145 2014-07-01 17:00:33,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:1530: snap_create 2014-07-01 17:00:33,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 890 (ObjNum 683 snap 81 seq_num 683) dirty exists 2014-07-01 17:00:34,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1529: done (1 left) 2014-07-01 17:00:34,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:1530: done (0 left) 2014-07-01 17:00:34,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: write oid 47 current snap is 149 2014-07-01 17:00:34,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: seq_num 881 ranges {519994=769129,1817483=529237,2767187=552807} 2014-07-01 17:00:34,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0437250-47 from 519994 to 1289123 tid 1 2014-07-01 17:00:34,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0437250-47 from 1817483 to 2346720 tid 2 2014-07-01 17:00:34,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: writing mira0437250-47 from 2767187 to 3319994 tid 3 2014-07-01 17:00:34,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: read oid 438 snap 51 2014-07-01 17:00:34,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 1 to mira0437250-47 2014-07-01 17:00:34,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 2 to mira0437250-47 2014-07-01 17:00:34,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: read oid 307 snap -1 2014-07-01 17:00:34,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1534: snap_create 2014-07-01 17:00:34,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 3 to mira0437250-47 2014-07-01 17:00:34,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 5 to mira0437250-47 2014-07-01 17:00:34,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: finishing write tid 6 to mira0437250-47 2014-07-01 17:00:34,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 733 (ObjNum 881 snap 149 seq_num 881) dirty exists 2014-07-01 17:00:34,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: expect (ObjNum 437 snap 0 seq_num 437) 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: finishing write tid 5 to mira0435601-70 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 70 v 283 (ObjNum 1868 snap 0 seq_num 1868) dirty exists 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 1 to mira0435601-161 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 2 to mira0435601-161 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 3 to mira0435601-161 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 4 to mira0435601-161 2014-07-01 17:00:34,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 5 to mira0435601-161 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 7 to mira0435601-161 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: finishing write tid 8 to mira0435601-161 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 181 (ObjNum 1869 snap 0 seq_num 1869) dirty exists 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: done (8 left) 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: done (7 left) 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: done (6 left) 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: done (5 left) 2014-07-01 17:00:34,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: done (4 left) 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: done (3 left) 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: done (2 left) 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: done (1 left) 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2984: done (0 left) 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2985: read oid 82 snap -1 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 17:00:34,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: seq_num 1871 ranges {0=299008} 2014-07-01 17:00:34,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: writing mira0435601-250 from 0 to 299008 tid 1 2014-07-01 17:00:34,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 17:00:35,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: seq_num 1872 ranges {4857856=729088,5586944=471040,6057984=331776} 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: writing mira0435601-176 from 4857856 to 5586944 tid 1 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: writing mira0435601-176 from 5586944 to 6057984 tid 2 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: writing mira0435601-176 from 6057984 to 6389760 tid 3 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 285 current snap is 0 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: seq_num 1873 ranges {2056192=598016,2654208=458752,3112960=344064} 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: writing mira0435601-285 from 2056192 to 2654208 tid 1 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: writing mira0435601-285 from 2654208 to 3112960 tid 2 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: writing mira0435601-285 from 3112960 to 3457024 tid 3 2014-07-01 17:00:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 470 current snap is 0 2014-07-01 17:00:35,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: seq_num 1874 ranges {2248704=471040,2719744=688128,3407872=114688} 2014-07-01 17:00:35,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: writing mira0435601-470 from 2248704 to 2719744 tid 1 2014-07-01 17:00:35,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: writing mira0435601-470 from 2719744 to 3407872 tid 2 2014-07-01 17:00:35,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: writing mira0435601-470 from 3407872 to 3522560 tid 3 2014-07-01 17:00:35,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2990: read oid 498 snap -1 2014-07-01 17:00:35,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: finishing write tid 1 to mira0435601-250 2014-07-01 17:00:35,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: finishing write tid 3 to mira0435601-250 2014-07-01 17:00:35,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: finishing write tid 4 to mira0435601-250 2014-07-01 17:00:35,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 157 (ObjNum 1871 snap 0 seq_num 1871) dirty exists 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: finishing write tid 1 to mira0435601-176 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2985: done (5 left) 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: done (4 left) 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2991: read oid 274 snap -1 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: finishing write tid 2 to mira0435601-176 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: finishing write tid 3 to mira0435601-176 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: finishing write tid 5 to mira0435601-176 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: finishing write tid 6 to mira0435601-176 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 251 (ObjNum 1872 snap 0 seq_num 1872) dirty exists 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 1 to mira0435601-285 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 2 to mira0435601-285 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 3 to mira0435601-285 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 5 to mira0435601-285 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: done (4 left) 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: read oid 451 snap -1 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 6 to mira0435601-285 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 185 (ObjNum 1873 snap 0 seq_num 1873) dirty exists 2014-07-01 17:00:35,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: finishing write tid 1 to mira0435601-470 2014-07-01 17:00:35,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: done (4 left) 2014-07-01 17:00:35,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2993: delete oid 9 current snap is 0 2014-07-01 17:00:35,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: finishing write tid 2 to mira0435601-470 2014-07-01 17:00:35,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2990: done (4 left) 2014-07-01 17:00:35,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2993: done (3 left) 2014-07-01 17:00:35,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2994: read oid 286 snap -1 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: read oid 394 snap -1 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2996: delete oid 405 current snap is 0 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2996: done (5 left) 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: read oid 40 snap -1 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 417 current snap is 0 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: seq_num 1877 ranges {0=692224} 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: writing mira0435601-417 from 0 to 692224 tid 1 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2999: delete oid 310 current snap is 0 2014-07-01 17:00:35,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2991: expect (ObjNum 1831 snap 0 seq_num 1831) 2014-07-01 17:00:35,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1531: done (3 left) 2014-07-01 17:00:35,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:1532: done (2 left) 2014-07-01 17:00:35,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1533: done (1 left) 2014-07-01 17:00:35,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1534: done (0 left) 2014-07-01 17:00:35,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: write oid 214 current snap is 150 2014-07-01 17:00:35,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: seq_num 882 ranges {615224=492358,1682016=599432,2967389=47835} 2014-07-01 17:00:35,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: writing mira0437250-214 from 615224 to 1107582 tid 1 2014-07-01 17:00:35,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: writing mira0437250-214 from 1682016 to 2281448 tid 2 2014-07-01 17:00:35,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: writing mira0437250-214 from 2967389 to 3015224 tid 3 2014-07-01 17:00:35,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: write oid 424 current snap is 150 2014-07-01 17:00:35,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: seq_num 883 ranges {559030=747810,1927348=553275,3184486=574544} 2014-07-01 17:00:35,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: writing mira0437250-424 from 559030 to 1306840 tid 1 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: writing mira0437250-424 from 1927348 to 2480623 tid 2 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: writing mira0437250-424 from 3184486 to 3759030 tid 3 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: write oid 147 current snap is 150 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: seq_num 884 ranges {620211=400000} 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: writing mira0437250-147 from 620211 to 1020211 tid 1 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: read oid 412 snap -1 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: finishing write tid 1 to mira0437250-214 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: finishing write tid 2 to mira0437250-214 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: finishing write tid 3 to mira0437250-214 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: finishing write tid 5 to mira0437250-214 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: finishing write tid 6 to mira0437250-214 2014-07-01 17:00:35,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 486 (ObjNum 882 snap 150 seq_num 882) dirty exists 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 1 to mira0437250-424 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 2 to mira0437250-424 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 3 to mira0437250-424 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 5 to mira0437250-424 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: finishing write tid 6 to mira0437250-424 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 424 v 903 (ObjNum 883 snap 150 seq_num 883) dirty exists 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1535: done (3 left) 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1536: done (2 left) 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1539: delete oid 156 current snap is 150 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1539: done (2 left) 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: rollback oid 58 current snap is 150 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 58 to 113 2014-07-01 17:00:35,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1541: snap_create 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 524 (ObjNum 57 snap 0 seq_num 57) dirty exists 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: finishing write tid 1 to mira0437250-147 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: finishing write tid 3 to mira0437250-147 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: finishing write tid 4 to mira0437250-147 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 557 (ObjNum 884 snap 150 seq_num 884) dirty exists 2014-07-01 17:00:35,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: expect (ObjNum 411 snap 0 seq_num 411) 2014-07-01 17:00:35,743.743 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:00:36,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: expect (ObjNum 1488 snap 0 seq_num 1488) 2014-07-01 17:00:36,451.451 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 10, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.012124', u'skew': u'0.001188', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.006135', u'skew': u'0.001156', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 07:59:44.725350', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 881002872, u'health': u'HEALTH_OK', u'kb_used': 15778180, u'store_stats': {u'bytes_total': 5869066, u'bytes_log': 983040, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4820474}}, {u'last_updated': u'2014-07-01 07:59:54.543144', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885210588, u'health': u'HEALTH_OK', u'kb_used': 11570464, u'store_stats': {u'bytes_total': 6916736, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4819568}}, {u'last_updated': u'2014-07-01 08:00:11.325798', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 885178984, u'health': u'HEALTH_OK', u'kb_used': 11602068, u'store_stats': {u'bytes_total': 10062471, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4819575}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 34 > pgp_num 24'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 56 > pgp_num 46'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_2 pg_num 36 > pgp_num 16'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 454, u'data_bytes': 2630565701, u'bytes_used': 6166114304, u'version': 2195, u'pgs_by_state': [{u'count': 454, u'state_name': u'active+clean'}], u'bytes_avail': 5992132931584}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 946, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:00:36,451.451 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:00:36,451.451 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:00:36,451.451 INFO:teuthology.task.thrashosds.thrasher:Removing osd 0, in_osds are: [3, 0, 5, 4, 1, 2] 2014-07-01 17:00:36,452.452 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 0' 2014-07-01 17:00:36,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1537: done (3 left) 2014-07-01 17:00:36,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1538: done (2 left) 2014-07-01 17:00:36,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1540: done (1 left) 2014-07-01 17:00:36,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1541: done (0 left) 2014-07-01 17:00:36,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: write oid 86 current snap is 151 2014-07-01 17:00:36,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: seq_num 886 ranges {660271=567095,1460270=1} 2014-07-01 17:00:36,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: writing mira0437250-86 from 660271 to 1227366 tid 1 2014-07-01 17:00:36,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: writing mira0437250-86 from 1460270 to 1460271 tid 2 2014-07-01 17:00:36,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: write oid 37 current snap is 151 2014-07-01 17:00:36,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: seq_num 887 ranges {756524=400000} 2014-07-01 17:00:36,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: writing mira0437250-37 from 756524 to 1156524 tid 1 2014-07-01 17:00:36,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: read oid 159 snap 85 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1545: read oid 17 snap 34 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1546: snap_remove snap 135 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 1 to mira0437250-86 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 2 to mira0437250-86 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 4 to mira0437250-86 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: finishing write tid 5 to mira0437250-86 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 320 (ObjNum 886 snap 151 seq_num 886) dirty exists 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: finishing write tid 1 to mira0437250-37 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: finishing write tid 3 to mira0437250-37 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: finishing write tid 4 to mira0437250-37 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 516 (ObjNum 887 snap 151 seq_num 887) dirty exists 2014-07-01 17:00:36,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1545: expect (ObjNum 535 snap 14 seq_num 535) 2014-07-01 17:00:36,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: finishing write tid 3 to mira0435601-470 2014-07-01 17:00:36,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: finishing write tid 5 to mira0435601-470 2014-07-01 17:00:36,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 17:00:36,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: finishing write tid 1 to mira0435601-417 2014-07-01 17:00:36,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: finishing write tid 3 to mira0435601-417 2014-07-01 17:00:36,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: finishing write tid 4 to mira0435601-417 2014-07-01 17:00:36,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 150 (ObjNum 1877 snap 0 seq_num 1877) dirty exists 2014-07-01 17:00:36,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:2994: expect (ObjNum 736 snap 0 seq_num 736) 2014-07-01 17:00:36,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: expect (ObjNum 158 snap 0 seq_num 158) 2014-07-01 17:00:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1542: done (4 left) 2014-07-01 17:00:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1543: done (3 left) 2014-07-01 17:00:37,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:1544: done (2 left) 2014-07-01 17:00:37,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1545: done (1 left) 2014-07-01 17:00:37,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:1546: done (0 left) 2014-07-01 17:00:37,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:1547: delete oid 144 current snap is 151 2014-07-01 17:00:37,529.529 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.0. 2014-07-01 17:00:38,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: expect (ObjNum 1778 snap 0 seq_num 1778) 2014-07-01 17:00:38,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1547: done (0 left) 2014-07-01 17:00:38,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1548: snap_remove snap 34 2014-07-01 17:00:39,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1548: done (0 left) 2014-07-01 17:00:39,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: snap_remove snap 141 2014-07-01 17:00:40,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:1549: done (0 left) 2014-07-01 17:00:40,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:1550: delete oid 472 current snap is 151 2014-07-01 17:00:42,542.542 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2] out_osds: [0] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:00:42,542.542 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:00:42,543.543 INFO:teuthology.task.thrashosds.thrasher:Adding osd 0 2014-07-01 17:00:42,544.544 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 0' 2014-07-01 17:00:42,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: finishing write tid 6 to mira0435601-470 2014-07-01 17:00:42,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 242 (ObjNum 1874 snap 0 seq_num 1874) dirty exists 2014-07-01 17:00:42,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: done (7 left) 2014-07-01 17:00:42,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2991: done (6 left) 2014-07-01 17:00:42,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: done (5 left) 2014-07-01 17:00:42,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2994: done (4 left) 2014-07-01 17:00:42,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: done (3 left) 2014-07-01 17:00:42,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: done (2 left) 2014-07-01 17:00:42,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: done (1 left) 2014-07-01 17:00:42,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2999: done (0 left) 2014-07-01 17:00:42,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 319 current snap is 0 2014-07-01 17:00:42,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: seq_num 1879 ranges {4358144=589824,4947968=495616,5443584=532480,5976064=176128} 2014-07-01 17:00:42,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: writing mira0435601-319 from 4358144 to 4947968 tid 1 2014-07-01 17:00:43,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: writing mira0435601-319 from 4947968 to 5443584 tid 2 2014-07-01 17:00:43,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: writing mira0435601-319 from 5443584 to 5976064 tid 3 2014-07-01 17:00:43,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: writing mira0435601-319 from 5976064 to 6152192 tid 4 2014-07-01 17:00:43,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3001: delete oid 165 current snap is 0 2014-07-01 17:00:43,895.895 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.0. 2014-07-01 17:00:43,907.907 INFO:teuthology.task.thrashosds.thrasher:Added osd 0 2014-07-01 17:00:44,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1550: done (0 left) 2014-07-01 17:00:44,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:1551: snap_remove snap 85 2014-07-01 17:00:44,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 1 to mira0435601-319 2014-07-01 17:00:44,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 2 to mira0435601-319 2014-07-01 17:00:44,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 3 to mira0435601-319 2014-07-01 17:00:44,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 4 to mira0435601-319 2014-07-01 17:00:44,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 6 to mira0435601-319 2014-07-01 17:00:44,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: finishing write tid 7 to mira0435601-319 2014-07-01 17:00:44,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 180 (ObjNum 1879 snap 0 seq_num 1879) dirty exists 2014-07-01 17:00:44,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1551: done (0 left) 2014-07-01 17:00:44,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: rollback oid 198 current snap is 151 2014-07-01 17:00:44,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 198 to 150 2014-07-01 17:00:44,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: snap_create 2014-07-01 17:00:44,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 502 (ObjNum 730 snap 98 seq_num 730) dirty exists 2014-07-01 17:00:45,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: done (1 left) 2014-07-01 17:00:45,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3001: done (0 left) 2014-07-01 17:00:45,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 38 current snap is 0 2014-07-01 17:00:45,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: seq_num 1881 ranges {1552384=540672,2093056=663552,2756608=139264} 2014-07-01 17:00:45,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0435601-38 from 1552384 to 2093056 tid 1 2014-07-01 17:00:45,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0435601-38 from 2093056 to 2756608 tid 2 2014-07-01 17:00:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0435601-38 from 2756608 to 2895872 tid 3 2014-07-01 17:00:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3003: read oid 424 snap -1 2014-07-01 17:00:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 105 current snap is 0 2014-07-01 17:00:45,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: seq_num 1882 ranges {2646016=409600,3055616=491520,3547136=311296} 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0435601-105 from 2646016 to 3055616 tid 1 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0435601-105 from 3055616 to 3547136 tid 2 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0435601-105 from 3547136 to 3858432 tid 3 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: read oid 449 snap -1 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 1 to mira0435601-38 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3003: done (3 left) 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: read oid 208 snap -1 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3007: read oid 326 snap -1 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3008: read oid 116 snap -1 2014-07-01 17:00:45,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 49 current snap is 0 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: seq_num 1883 ranges {1355776=684032,2039808=401408} 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: writing mira0435601-49 from 1355776 to 2039808 tid 1 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: writing mira0435601-49 from 2039808 to 2441216 tid 2 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: seq_num 1884 ranges {0=712704,712704=659456,1372160=651264,2023424=294912} 2014-07-01 17:00:45,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: writing mira0435601-273 from 0 to 712704 tid 1 2014-07-01 17:00:45,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: writing mira0435601-273 from 712704 to 1372160 tid 2 2014-07-01 17:00:45,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: writing mira0435601-273 from 1372160 to 2023424 tid 3 2014-07-01 17:00:45,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: writing mira0435601-273 from 2023424 to 2318336 tid 4 2014-07-01 17:00:45,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3008: expect (ObjNum 900 snap 0 seq_num 900) 2014-07-01 17:00:45,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1552: done (1 left) 2014-07-01 17:00:45,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1553: done (0 left) 2014-07-01 17:00:45,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: rollback oid 352 current snap is 152 2014-07-01 17:00:45,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 352 to 118 2014-07-01 17:00:45,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: read oid 110 snap 149 2014-07-01 17:00:45,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1556: rollback oid 183 current snap is 152 2014-07-01 17:00:45,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 183 to 140 2014-07-01 17:00:45,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: write oid 321 current snap is 152 2014-07-01 17:00:45,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: seq_num 890 ranges {455718=641832,1534337=547490,2654121=201597} 2014-07-01 17:00:45,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: writing mira0437250-321 from 455718 to 1097550 tid 1 2014-07-01 17:00:45,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: writing mira0437250-321 from 1534337 to 2081827 tid 2 2014-07-01 17:00:45,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: writing mira0437250-321 from 2654121 to 2855718 tid 3 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: write oid 174 current snap is 152 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: seq_num 891 ranges {649122=1} 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: writing mira0437250-174 from 649122 to 649123 tid 1 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: snap_remove snap 71 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 352 v 0 (ObjNum -1595748976 snap 32767 seq_num 46250256) dirty dne 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 463 (ObjNum 182 snap 0 seq_num 182) dirty exists 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 1 to mira0437250-321 2014-07-01 17:00:46,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: expect (ObjNum 109 snap 0 seq_num 109) 2014-07-01 17:00:46,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: expect (ObjNum 1231 snap 0 seq_num 1231) 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 2 to mira0437250-321 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 3 to mira0437250-321 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 5 to mira0437250-321 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: finishing write tid 1 to mira0437250-174 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: finishing write tid 3 to mira0437250-174 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: finishing write tid 4 to mira0437250-174 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 513 (ObjNum 891 snap 152 seq_num 891) dirty exists 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: finishing write tid 6 to mira0437250-321 2014-07-01 17:00:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 507 (ObjNum 890 snap 152 seq_num 890) dirty exists 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1554: done (5 left) 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1555: done (4 left) 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1556: done (3 left) 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1557: done (2 left) 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1558: done (1 left) 2014-07-01 17:00:46,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1559: done (0 left) 2014-07-01 17:00:46,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: write oid 215 current snap is 152 2014-07-01 17:00:46,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: seq_num 892 ranges {345376=1} 2014-07-01 17:00:46,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: writing mira0437250-215 from 345376 to 345377 tid 1 2014-07-01 17:00:46,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: write oid 496 current snap is 152 2014-07-01 17:00:46,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: seq_num 893 ranges {556958=1} 2014-07-01 17:00:46,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: writing mira0437250-496 from 556958 to 556959 tid 1 2014-07-01 17:00:47,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: write oid 16 current snap is 152 2014-07-01 17:00:47,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: seq_num 894 ranges {750365=565146,2017874=332491} 2014-07-01 17:00:47,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: writing mira0437250-16 from 750365 to 1315511 tid 1 2014-07-01 17:00:47,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: writing mira0437250-16 from 2017874 to 2350365 tid 2 2014-07-01 17:00:47,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1563: snap_create 2014-07-01 17:00:47,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: finishing write tid 1 to mira0437250-215 2014-07-01 17:00:47,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: finishing write tid 3 to mira0437250-215 2014-07-01 17:00:47,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: finishing write tid 4 to mira0437250-215 2014-07-01 17:00:47,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 789 (ObjNum 892 snap 152 seq_num 892) dirty exists 2014-07-01 17:00:47,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 1 to mira0435601-105 2014-07-01 17:00:47,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 2 to mira0435601-105 2014-07-01 17:00:47,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 3 to mira0435601-105 2014-07-01 17:00:47,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 5 to mira0435601-105 2014-07-01 17:00:47,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 6 to mira0435601-105 2014-07-01 17:00:47,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 161 (ObjNum 1882 snap 0 seq_num 1882) dirty exists 2014-07-01 17:00:47,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3007: expect (ObjNum 1849 snap 0 seq_num 1849) 2014-07-01 17:00:47,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: finishing write tid 1 to mira0437250-16 2014-07-01 17:00:47,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: finishing write tid 2 to mira0437250-16 2014-07-01 17:00:47,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: finishing write tid 4 to mira0437250-16 2014-07-01 17:00:47,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: finishing write tid 5 to mira0437250-16 2014-07-01 17:00:47,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 475 (ObjNum 894 snap 152 seq_num 894) dirty exists 2014-07-01 17:00:47,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: finishing write tid 1 to mira0437250-496 2014-07-01 17:00:47,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: finishing write tid 3 to mira0437250-496 2014-07-01 17:00:47,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: finishing write tid 4 to mira0437250-496 2014-07-01 17:00:47,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 447 (ObjNum 893 snap 152 seq_num 893) dirty exists 2014-07-01 17:00:47,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: finishing write tid 1 to mira0435601-49 2014-07-01 17:00:47,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: expect (ObjNum 1769 snap 0 seq_num 1769) 2014-07-01 17:00:47,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1560: done (3 left) 2014-07-01 17:00:47,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1561: done (2 left) 2014-07-01 17:00:47,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1562: done (1 left) 2014-07-01 17:00:47,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1563: done (0 left) 2014-07-01 17:00:47,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1564: rollback oid 264 current snap is 153 2014-07-01 17:00:47,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 264 to 112 2014-07-01 17:00:47,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: write oid 159 current snap is 153 2014-07-01 17:00:47,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: seq_num 895 ranges {445594=421622,1614629=430965} 2014-07-01 17:00:47,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: writing mira0437250-159 from 445594 to 867216 tid 1 2014-07-01 17:00:47,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: writing mira0437250-159 from 1614629 to 2045594 tid 2 2014-07-01 17:00:47,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:1566: snap_create 2014-07-01 17:00:48,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1058 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 17:00:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: finishing write tid 1 to mira0437250-159 2014-07-01 17:00:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: finishing write tid 2 to mira0437250-159 2014-07-01 17:00:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: finishing write tid 4 to mira0437250-159 2014-07-01 17:00:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: finishing write tid 5 to mira0437250-159 2014-07-01 17:00:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 159 v 570 (ObjNum 895 snap 153 seq_num 895) dirty exists 2014-07-01 17:00:48,907.907 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:00:48,908.908 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:00:48,908.908 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:00:48,908.908 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:00:48,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1564: done (2 left) 2014-07-01 17:00:48,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1565: done (1 left) 2014-07-01 17:00:48,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:1566: done (0 left) 2014-07-01 17:00:48,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: read oid 417 snap -1 2014-07-01 17:00:48,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: read oid 250 snap -1 2014-07-01 17:00:48,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: rollback oid 299 current snap is 154 2014-07-01 17:00:48,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 299 to 145 2014-07-01 17:00:48,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: snap_remove snap 113 2014-07-01 17:00:48,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 529 (ObjNum 515 snap 8 seq_num 515) dirty exists 2014-07-01 17:00:48,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: expect (ObjNum 249 snap 0 seq_num 249) 2014-07-01 17:00:49,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: expect (ObjNum 751 snap 109 seq_num 751) 2014-07-01 17:00:49,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: finishing write tid 2 to mira0435601-49 2014-07-01 17:00:49,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: finishing write tid 4 to mira0435601-49 2014-07-01 17:00:49,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: finishing write tid 5 to mira0435601-49 2014-07-01 17:00:49,537.537 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 49 v 291 (ObjNum 1883 snap 0 seq_num 1883) dirty exists 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 2 to mira0435601-38 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 3 to mira0435601-38 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 5 to mira0435601-38 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 1 to mira0435601-273 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 2 to mira0435601-273 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 3 to mira0435601-273 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 4 to mira0435601-273 2014-07-01 17:00:49,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 6 to mira0435601-273 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3011: read oid 456 snap -1 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: finishing write tid 7 to mira0435601-273 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 185 (ObjNum 1884 snap 0 seq_num 1884) dirty exists 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 6 to mira0435601-38 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 287 (ObjNum 1881 snap 0 seq_num 1881) dirty exists 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: done (8 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: done (7 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: done (6 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: done (5 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3007: done (4 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3008: done (3 left) 2014-07-01 17:00:49,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: done (2 left) 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: done (1 left) 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 249 current snap is 0 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: seq_num 1885 ranges {0=630784} 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: writing mira0435601-249 from 0 to 630784 tid 1 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3013: read oid 344 snap -1 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: seq_num 1886 ranges {7630848=659456,8290304=720896,9011200=483328} 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: writing mira0435601-313 from 7630848 to 8290304 tid 1 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: writing mira0435601-313 from 8290304 to 9011200 tid 2 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: writing mira0435601-313 from 9011200 to 9494528 tid 3 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: delete oid 489 current snap is 0 2014-07-01 17:00:49,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3013: expect (ObjNum 1448 snap 0 seq_num 1448) 2014-07-01 17:00:50,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:3011: expect (ObjNum 1770 snap 0 seq_num 1770) 2014-07-01 17:00:51,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1567: done (3 left) 2014-07-01 17:00:51,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1568: done (2 left) 2014-07-01 17:00:51,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1569: done (1 left) 2014-07-01 17:00:51,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1570: done (0 left) 2014-07-01 17:00:51,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: write oid 355 current snap is 154 2014-07-01 17:00:51,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: seq_num 896 ranges {541848=544851,1596538=145310} 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: writing mira0437250-355 from 541848 to 1086699 tid 1 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: writing mira0437250-355 from 1596538 to 1741848 tid 2 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: delete oid 416 current snap is 154 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1572: done (1 left) 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: snap_remove snap 150 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: finishing write tid 1 to mira0437250-355 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: finishing write tid 2 to mira0437250-355 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: finishing write tid 4 to mira0437250-355 2014-07-01 17:00:51,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: finishing write tid 5 to mira0437250-355 2014-07-01 17:00:51,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 411 (ObjNum 896 snap 154 seq_num 896) dirty exists 2014-07-01 17:00:51,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:1571: done (1 left) 2014-07-01 17:00:51,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:1573: done (0 left) 2014-07-01 17:00:51,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: write oid 296 current snap is 154 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: seq_num 898 ranges {545812=639883,1636258=645947,3008627=551005,3745811=1} 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: writing mira0437250-296 from 545812 to 1185695 tid 1 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: writing mira0437250-296 from 1636258 to 2282205 tid 2 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: writing mira0437250-296 from 3008627 to 3559632 tid 3 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: writing mira0437250-296 from 3745811 to 3745812 tid 4 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: delete oid 298 current snap is 154 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 1 to mira0437250-296 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 2 to mira0437250-296 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1575: done (1 left) 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: read oid 261 snap -1 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: snap_create 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 3 to mira0437250-296 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 4 to mira0437250-296 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 6 to mira0437250-296 2014-07-01 17:00:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: finishing write tid 7 to mira0437250-296 2014-07-01 17:00:51,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 586 (ObjNum 898 snap 154 seq_num 898) dirty exists 2014-07-01 17:00:51,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: expect (ObjNum 260 snap 0 seq_num 260) 2014-07-01 17:00:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1574: done (2 left) 2014-07-01 17:00:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1576: done (1 left) 2014-07-01 17:00:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1577: done (0 left) 2014-07-01 17:00:52,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1578: snap_create 2014-07-01 17:00:53,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1578: done (0 left) 2014-07-01 17:00:53,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:1579: read oid 204 snap -1 2014-07-01 17:00:53,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:1580: snap_create 2014-07-01 17:00:53,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:1579: expect (ObjNum 203 snap 0 seq_num 203) 2014-07-01 17:00:53,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 1 to mira0435601-249 2014-07-01 17:00:53,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 3 to mira0435601-249 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: finishing write tid 1 to mira0435601-313 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: finishing write tid 2 to mira0435601-313 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: finishing write tid 3 to mira0435601-313 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: finishing write tid 5 to mira0435601-313 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3011: done (4 left) 2014-07-01 17:00:53,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3013: done (3 left) 2014-07-01 17:00:53,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: done (2 left) 2014-07-01 17:00:53,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3016: read oid 407 snap -1 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: finishing write tid 6 to mira0435601-313 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 215 (ObjNum 1886 snap 0 seq_num 1886) dirty exists 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 4 to mira0435601-249 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 230 (ObjNum 1885 snap 0 seq_num 1885) dirty exists 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: done (2 left) 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: done (1 left) 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3017: read oid 197 snap -1 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 101 current snap is 0 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: seq_num 1888 ranges {1851392=802816,2654208=483328,3137536=319488} 2014-07-01 17:00:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: writing mira0435601-101 from 1851392 to 2654208 tid 1 2014-07-01 17:00:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: writing mira0435601-101 from 2654208 to 3137536 tid 2 2014-07-01 17:00:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: writing mira0435601-101 from 3137536 to 3457024 tid 3 2014-07-01 17:00:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: read oid 371 snap -1 2014-07-01 17:00:54,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3016: expect (ObjNum 1531 snap 0 seq_num 1531) 2014-07-01 17:00:54,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 1 to mira0435601-101 2014-07-01 17:00:54,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3017: expect (ObjNum 1605 snap 0 seq_num 1605) 2014-07-01 17:00:55,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1579: done (1 left) 2014-07-01 17:00:55,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1580: done (0 left) 2014-07-01 17:00:55,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:1581: snap_create 2014-07-01 17:00:55,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1581: done (0 left) 2014-07-01 17:00:55,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: read oid 402 snap -1 2014-07-01 17:00:55,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: write oid 417 current snap is 158 2014-07-01 17:00:55,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: seq_num 900 ranges {185870=1} 2014-07-01 17:00:55,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: writing mira0437250-417 from 185870 to 185871 tid 1 2014-07-01 17:00:55,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1584: delete oid 128 current snap is 158 2014-07-01 17:00:55,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 1 to mira0437250-417 2014-07-01 17:00:55,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 3 to mira0437250-417 2014-07-01 17:00:55,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1582: done (2 left) 2014-07-01 17:00:55,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1584: done (1 left) 2014-07-01 17:00:55,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: rollback oid 98 current snap is 158 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 98 to 118 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1586: delete oid 391 current snap is 158 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 605 (ObjNum 97 snap 0 seq_num 97) dirty exists 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: finishing write tid 4 to mira0437250-417 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 450 (ObjNum 900 snap 158 seq_num 900) dirty exists 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1583: done (2 left) 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1585: done (1 left) 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1586: done (0 left) 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1587: rollback oid 130 current snap is 158 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 130 to 145 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: delete oid 227 current snap is 158 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 469 (ObjNum 129 snap 0 seq_num 129) dirty exists 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1587: done (1 left) 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1588: done (0 left) 2014-07-01 17:00:55,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: write oid 317 current snap is 158 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: seq_num 904 ranges {631942=542901,1947034=453614,2631941=1} 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: writing mira0437250-317 from 631942 to 1174843 tid 1 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: writing mira0437250-317 from 1947034 to 2400648 tid 2 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: writing mira0437250-317 from 2631941 to 2631942 tid 3 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1590: snap_remove snap 153 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: finishing write tid 1 to mira0437250-317 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: finishing write tid 2 to mira0437250-317 2014-07-01 17:00:55,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: finishing write tid 3 to mira0437250-317 2014-07-01 17:00:55,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: finishing write tid 5 to mira0437250-317 2014-07-01 17:00:55,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: finishing write tid 6 to mira0437250-317 2014-07-01 17:00:55,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 849 (ObjNum 904 snap 158 seq_num 904) dirty exists 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 2 to mira0435601-101 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 3 to mira0435601-101 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 5 to mira0435601-101 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 6 to mira0435601-101 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 153 (ObjNum 1888 snap 0 seq_num 1888) dirty exists 2014-07-01 17:00:56,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: expect (ObjNum 1848 snap 0 seq_num 1848) 2014-07-01 17:00:56,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:1589: done (1 left) 2014-07-01 17:00:56,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:1590: done (0 left) 2014-07-01 17:00:56,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:1591: delete oid 391 current snap is 158 2014-07-01 17:00:56,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1591: done (0 left) 2014-07-01 17:00:56,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: write oid 66 current snap is 158 2014-07-01 17:00:56,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: seq_num 906 ranges {670978=639982,1740227=745429,3070977=1} 2014-07-01 17:00:56,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: writing mira0437250-66 from 670978 to 1310960 tid 1 2014-07-01 17:00:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: writing mira0437250-66 from 1740227 to 2485656 tid 2 2014-07-01 17:00:56,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: writing mira0437250-66 from 3070977 to 3070978 tid 3 2014-07-01 17:00:56,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1593: read oid 104 snap 151 2014-07-01 17:00:56,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: finishing write tid 1 to mira0437250-66 2014-07-01 17:00:56,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: read oid 470 snap -1 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1595: rollback oid 157 current snap is 158 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 157 to 148 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: rollback oid 376 current snap is 158 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 376 to 151 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: snap_remove snap 79 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 0 (ObjNum -1829423232 snap 32691 seq_num 1703937) dirty dne 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 0 (ObjNum 16145136 snap 0 seq_num 32) dirty dne 2014-07-01 17:00:56,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: expect (ObjNum 469 snap 0 seq_num 469) 2014-07-01 17:00:57,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: finishing write tid 2 to mira0437250-66 2014-07-01 17:00:57,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: finishing write tid 3 to mira0437250-66 2014-07-01 17:00:57,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: finishing write tid 5 to mira0437250-66 2014-07-01 17:00:57,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: finishing write tid 6 to mira0437250-66 2014-07-01 17:00:57,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 697 (ObjNum 906 snap 158 seq_num 906) dirty exists 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1592: done (5 left) 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1593: done (4 left) 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1594: done (3 left) 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1595: done (2 left) 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1596: done (1 left) 2014-07-01 17:00:57,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1597: done (0 left) 2014-07-01 17:00:57,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1598: snap_remove snap 154 2014-07-01 17:00:57,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3016: done (3 left) 2014-07-01 17:00:57,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3017: done (2 left) 2014-07-01 17:00:57,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: done (1 left) 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: done (0 left) 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 166 current snap is 0 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: seq_num 1889 ranges {5087232=479232,5566464=471040,6037504=331776} 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: writing mira0435601-166 from 5087232 to 5566464 tid 1 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: writing mira0435601-166 from 5566464 to 6037504 tid 2 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: writing mira0435601-166 from 6037504 to 6369280 tid 3 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3021: read oid 109 snap -1 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 15 current snap is 0 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: seq_num 1890 ranges {4071424=753664,4825088=401408} 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: writing mira0435601-15 from 4071424 to 4825088 tid 1 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: writing mira0435601-15 from 4825088 to 5226496 tid 2 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: read oid 268 snap -1 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: finishing write tid 1 to mira0435601-166 2014-07-01 17:00:57,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: finishing write tid 2 to mira0435601-166 2014-07-01 17:00:57,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3024: read oid 311 snap -1 2014-07-01 17:00:57,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 17:00:57,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: seq_num 1891 ranges {1335296=622592,1957888=401408} 2014-07-01 17:00:57,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: writing mira0435601-444 from 1335296 to 1957888 tid 1 2014-07-01 17:00:57,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: writing mira0435601-444 from 1957888 to 2359296 tid 2 2014-07-01 17:00:58,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 71 current snap is 0 2014-07-01 17:00:58,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: seq_num 1892 ranges {4923392=651264,5574656=466944,6041600=700416,6742016=438272} 2014-07-01 17:00:58,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0435601-71 from 4923392 to 5574656 tid 1 2014-07-01 17:00:58,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0435601-71 from 5574656 to 6041600 tid 2 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0435601-71 from 6041600 to 6742016 tid 3 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0435601-71 from 6742016 to 7180288 tid 4 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 146 current snap is 0 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: seq_num 1893 ranges {3465216=520192,3985408=450560,4435968=774144,5210112=380928} 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: writing mira0435601-146 from 3465216 to 3985408 tid 1 2014-07-01 17:00:58,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: writing mira0435601-146 from 3985408 to 4435968 tid 2 2014-07-01 17:00:58,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: writing mira0435601-146 from 4435968 to 5210112 tid 3 2014-07-01 17:00:58,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: writing mira0435601-146 from 5210112 to 5591040 tid 4 2014-07-01 17:00:58,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:3028: read oid 207 snap -1 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: finishing write tid 1 to mira0435601-15 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: finishing write tid 2 to mira0435601-15 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: finishing write tid 4 to mira0435601-15 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: finishing write tid 5 to mira0435601-15 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 156 (ObjNum 1890 snap 0 seq_num 1890) dirty exists 2014-07-01 17:00:58,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3021: expect (ObjNum 1557 snap 0 seq_num 1557) 2014-07-01 17:00:58,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1598: done (0 left) 2014-07-01 17:00:58,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1599: read oid 115 snap -1 2014-07-01 17:00:58,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: read oid 27 snap -1 2014-07-01 17:00:58,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: write oid 391 current snap is 158 2014-07-01 17:00:58,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: seq_num 907 ranges {332158=1} 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: writing mira0437250-391 from 332158 to 332159 tid 1 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: write oid 334 current snap is 158 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: seq_num 908 ranges {27388=1} 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: writing mira0437250-334 from 27388 to 27389 tid 1 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1603: read oid 462 snap 152 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1604: read oid 443 snap 143 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: read oid 353 snap -1 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: rollback oid 329 current snap is 158 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 329 to 112 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: snap_remove snap 115 2014-07-01 17:00:58,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: expect (ObjNum 647 snap 67 seq_num 647) 2014-07-01 17:00:58,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: finishing write tid 1 to mira0437250-391 2014-07-01 17:00:58,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: finishing write tid 3 to mira0437250-391 2014-07-01 17:00:58,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: finishing write tid 4 to mira0437250-391 2014-07-01 17:00:58,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 630 (ObjNum 907 snap 158 seq_num 907) dirty exists 2014-07-01 17:00:58,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: expect (ObjNum 352 snap 0 seq_num 352) 2014-07-01 17:00:59,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: finishing write tid 1 to mira0437250-334 2014-07-01 17:00:59,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:1603: expect (ObjNum 461 snap 0 seq_num 461) 2014-07-01 17:00:59,468.468 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 34' 2014-07-01 17:00:59,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: finishing write tid 3 to mira0437250-334 2014-07-01 17:00:59,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 329 v 822 (ObjNum 328 snap 0 seq_num 328) dirty exists 2014-07-01 17:00:59,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: finishing write tid 4 to mira0437250-334 2014-07-01 17:00:59,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 505 (ObjNum 908 snap 158 seq_num 908) dirty exists 2014-07-01 17:00:59,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:1599: expect (ObjNum 780 snap 124 seq_num 780) 2014-07-01 17:01:00,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:1599: done (8 left) 2014-07-01 17:01:00,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1600: done (7 left) 2014-07-01 17:01:00,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1601: done (6 left) 2014-07-01 17:01:00,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1602: done (5 left) 2014-07-01 17:01:00,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1603: done (4 left) 2014-07-01 17:01:00,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1604: done (3 left) 2014-07-01 17:01:00,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1605: done (2 left) 2014-07-01 17:01:00,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:1606: done (1 left) 2014-07-01 17:01:00,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:1607: done (0 left) 2014-07-01 17:01:00,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: read oid 496 snap -1 2014-07-01 17:01:00,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: snap_create 2014-07-01 17:01:00,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: expect (ObjNum 893 snap 152 seq_num 893) 2014-07-01 17:01:00,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1608: done (1 left) 2014-07-01 17:01:00,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1609: done (0 left) 2014-07-01 17:01:00,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:1610: rollback oid 369 current snap is 159 2014-07-01 17:01:00,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 369 to 157 2014-07-01 17:01:00,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: read oid 241 snap -1 2014-07-01 17:01:00,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: snap_remove snap 132 2014-07-01 17:01:00,789.789 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 34 2014-07-01 17:01:00,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 369 v 469 (ObjNum 368 snap 0 seq_num 368) dirty exists 2014-07-01 17:01:00,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: expect (ObjNum 240 snap 0 seq_num 240) 2014-07-01 17:01:00,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: finishing write tid 3 to mira0435601-166 2014-07-01 17:01:00,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: finishing write tid 5 to mira0435601-166 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: finishing write tid 6 to mira0435601-166 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 279 (ObjNum 1889 snap 0 seq_num 1889) dirty exists 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: finishing write tid 1 to mira0435601-444 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: finishing write tid 2 to mira0435601-444 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: finishing write tid 4 to mira0435601-444 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: done (8 left) 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3021: done (7 left) 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: done (6 left) 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3029: read oid 123 snap -1 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 1 to mira0435601-71 2014-07-01 17:01:00,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: expect (ObjNum 1726 snap 0 seq_num 1726) 2014-07-01 17:01:01,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:1610: done (2 left) 2014-07-01 17:01:01,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:1611: done (1 left) 2014-07-01 17:01:01,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:1612: done (0 left) 2014-07-01 17:01:01,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: rollback oid 414 current snap is 159 2014-07-01 17:01:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 414 to 118 2014-07-01 17:01:01,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: read oid 328 snap 155 2014-07-01 17:01:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1615: rollback oid 315 current snap is 159 2014-07-01 17:01:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 315 to 157 2014-07-01 17:01:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: read oid 162 snap -1 2014-07-01 17:01:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1617: snap_create 2014-07-01 17:01:01,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: expect (ObjNum 327 snap 0 seq_num 327) 2014-07-01 17:01:03,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 317 (ObjNum 314 snap 0 seq_num 314) dirty exists 2014-07-01 17:01:03,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: expect (ObjNum 161 snap 0 seq_num 161) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 843 (ObjNum 762 snap 112 seq_num 762) dirty exists 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1613: done (4 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1614: done (3 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1615: done (2 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1616: done (1 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1617: done (0 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1618: delete oid 206 current snap is 160 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1618: done (0 left) 2014-07-01 17:01:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1619: snap_remove snap 151 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 2 to mira0435601-71 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: finishing write tid 5 to mira0435601-444 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 233 (ObjNum 1891 snap 0 seq_num 1891) dirty exists 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 3 to mira0435601-71 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 4 to mira0435601-71 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 6 to mira0435601-71 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 7 to mira0435601-71 2014-07-01 17:01:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 238 (ObjNum 1892 snap 0 seq_num 1892) dirty exists 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 1 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 2 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 3 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: done (6 left) 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: done (5 left) 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: done (4 left) 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3028: done (3 left) 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3030: delete oid 43 current snap is 0 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 4 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 6 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: finishing write tid 7 to mira0435601-146 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 199 (ObjNum 1893 snap 0 seq_num 1893) dirty exists 2014-07-01 17:01:04,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3024: expect (ObjNum 1580 snap 0 seq_num 1580) 2014-07-01 17:01:04,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:1619: done (0 left) 2014-07-01 17:01:04,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: snap_create 2014-07-01 17:01:05,803.803 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:05,804.804 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:05,804.804 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 17:01:05,804.804 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:01:05,805.805 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:01:05,805.805 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 17:01:05,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1620: done (0 left) 2014-07-01 17:01:05,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:1621: snap_remove snap 118 2014-07-01 17:01:06,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1621: done (0 left) 2014-07-01 17:01:06,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: read oid 185 snap 155 2014-07-01 17:01:06,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1623: snap_remove snap 148 2014-07-01 17:01:06,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: expect (ObjNum 643 snap 66 seq_num 643) 2014-07-01 17:01:07,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1622: done (1 left) 2014-07-01 17:01:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1623: done (0 left) 2014-07-01 17:01:07,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: delete oid 500 current snap is 161 2014-07-01 17:01:07,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1624: done (0 left) 2014-07-01 17:01:07,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:1625: delete oid 6 current snap is 161 2014-07-01 17:01:07,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1625: done (0 left) 2014-07-01 17:01:07,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: read oid 337 snap 127 2014-07-01 17:01:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1627: snap_create 2014-07-01 17:01:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: expect (ObjNum 722 snap 97 seq_num 722) 2014-07-01 17:01:08,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3029: expect (ObjNum 1856 snap 0 seq_num 1856) 2014-07-01 17:01:08,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1626: done (1 left) 2014-07-01 17:01:08,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:1627: done (0 left) 2014-07-01 17:01:08,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: read oid 319 snap -1 2014-07-01 17:01:08,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:1629: delete oid 455 current snap is 162 2014-07-01 17:01:08,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:1629: done (1 left) 2014-07-01 17:01:08,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:1630: delete oid 400 current snap is 162 2014-07-01 17:01:09,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1630: done (1 left) 2014-07-01 17:01:09,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1631: delete oid 71 current snap is 162 2014-07-01 17:01:09,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: expect (ObjNum 318 snap 0 seq_num 318) 2014-07-01 17:01:10,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1628: done (1 left) 2014-07-01 17:01:10,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1631: done (0 left) 2014-07-01 17:01:10,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:1632: delete oid 206 current snap is 162 2014-07-01 17:01:10,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1632: done (0 left) 2014-07-01 17:01:10,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1633: rollback oid 288 current snap is 162 2014-07-01 17:01:10,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 288 to 112 2014-07-01 17:01:10,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1634: snap_create 2014-07-01 17:01:10,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 525 (ObjNum 755 snap 110 seq_num 755) dirty exists 2014-07-01 17:01:10,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3024: done (3 left) 2014-07-01 17:01:10,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: done (2 left) 2014-07-01 17:01:10,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3029: done (1 left) 2014-07-01 17:01:10,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:3030: done (0 left) 2014-07-01 17:01:10,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3031: read oid 421 snap -1 2014-07-01 17:01:10,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 180 current snap is 0 2014-07-01 17:01:10,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: seq_num 1895 ranges {589824=471040,1060864=667648,1728512=475136,2203648=61440} 2014-07-01 17:01:10,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: writing mira0435601-180 from 589824 to 1060864 tid 1 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: writing mira0435601-180 from 1060864 to 1728512 tid 2 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: writing mira0435601-180 from 1728512 to 2203648 tid 3 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: writing mira0435601-180 from 2203648 to 2265088 tid 4 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 99 current snap is 0 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: seq_num 1896 ranges {929792=741376,1671168=446464,2117632=356352} 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: writing mira0435601-99 from 929792 to 1671168 tid 1 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: writing mira0435601-99 from 1671168 to 2117632 tid 2 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: writing mira0435601-99 from 2117632 to 2473984 tid 3 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3034: read oid 125 snap -1 2014-07-01 17:01:10,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3031: expect (ObjNum 1421 snap 0 seq_num 1421) 2014-07-01 17:01:10,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1633: done (1 left) 2014-07-01 17:01:10,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:1634: done (0 left) 2014-07-01 17:01:10,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: write oid 124 current snap is 163 2014-07-01 17:01:10,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: seq_num 916 ranges {668907=617737,2035340=735701,3304988=563919} 2014-07-01 17:01:10,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: writing mira0437250-124 from 668907 to 1286644 tid 1 2014-07-01 17:01:10,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: writing mira0437250-124 from 2035340 to 2771041 tid 2 2014-07-01 17:01:10,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: writing mira0437250-124 from 3304988 to 3868907 tid 3 2014-07-01 17:01:10,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1636: read oid 231 snap -1 2014-07-01 17:01:10,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 1 to mira0437250-124 2014-07-01 17:01:10,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 2 to mira0437250-124 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: rollback oid 448 current snap is 163 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 448 to 140 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1638: snap_remove snap 157 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 748 (ObjNum 704 snap 93 seq_num 704) dirty exists 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 3 to mira0437250-124 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 5 to mira0437250-124 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: finishing write tid 6 to mira0437250-124 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 867 (ObjNum 916 snap 163 seq_num 916) dirty exists 2014-07-01 17:01:10,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:1636: expect (ObjNum 640 snap 65 seq_num 640) 2014-07-01 17:01:11,359.359 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:11,360.360 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:11,360.360 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 3 to 0.916641644777 2014-07-01 17:01:11,360.360 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 0.916641644777' 2014-07-01 17:01:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 1 to mira0435601-180 2014-07-01 17:01:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 2 to mira0435601-180 2014-07-01 17:01:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 3 to mira0435601-180 2014-07-01 17:01:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 4 to mira0435601-180 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 6 to mira0435601-180 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: finishing write tid 7 to mira0435601-180 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 296 (ObjNum 1895 snap 0 seq_num 1895) dirty exists 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3031: done (3 left) 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: done (2 left) 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3035: delete oid 298 current snap is 0 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: finishing write tid 1 to mira0435601-99 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: finishing write tid 2 to mira0435601-99 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: finishing write tid 3 to mira0435601-99 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: finishing write tid 5 to mira0435601-99 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: finishing write tid 6 to mira0435601-99 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 99 v 154 (ObjNum 1896 snap 0 seq_num 1896) dirty exists 2014-07-01 17:01:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3034: expect (ObjNum 1779 snap 0 seq_num 1779) 2014-07-01 17:01:11,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:1635: done (3 left) 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1636: done (2 left) 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1637: done (1 left) 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1638: done (0 left) 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: write oid 259 current snap is 163 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: seq_num 917 ranges {613736=1} 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: writing mira0437250-259 from 613736 to 613737 tid 1 2014-07-01 17:01:11,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: snap_remove snap 134 2014-07-01 17:01:12,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 1 to mira0437250-259 2014-07-01 17:01:12,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 3 to mira0437250-259 2014-07-01 17:01:12,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: finishing write tid 4 to mira0437250-259 2014-07-01 17:01:12,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 452 (ObjNum 917 snap 163 seq_num 917) dirty exists 2014-07-01 17:01:12,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1639: done (1 left) 2014-07-01 17:01:12,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1640: done (0 left) 2014-07-01 17:01:12,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1641: rollback oid 106 current snap is 163 2014-07-01 17:01:12,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 106 to 146 2014-07-01 17:01:12,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: read oid 51 snap -1 2014-07-01 17:01:12,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: read oid 133 snap -1 2014-07-01 17:01:12,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: snap_create 2014-07-01 17:01:12,894.894 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 0.916642 (8600732) 2014-07-01 17:01:13,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: expect (ObjNum 664 snap 75 seq_num 664) 2014-07-01 17:01:13,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: expect (ObjNum 132 snap 0 seq_num 132) 2014-07-01 17:01:13,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 919 (ObjNum 863 snap 145 seq_num 863) dirty exists 2014-07-01 17:01:13,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: done (2 left) 2014-07-01 17:01:13,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3034: done (1 left) 2014-07-01 17:01:13,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3035: done (0 left) 2014-07-01 17:01:13,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 443 current snap is 0 2014-07-01 17:01:13,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: seq_num 1898 ranges {2924544=692224,3616768=401408} 2014-07-01 17:01:13,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: writing mira0435601-443 from 2924544 to 3616768 tid 1 2014-07-01 17:01:13,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: writing mira0435601-443 from 3616768 to 4018176 tid 2 2014-07-01 17:01:13,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: read oid 13 snap -1 2014-07-01 17:01:13,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3038: delete oid 95 current snap is 0 2014-07-01 17:01:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3038: done (2 left) 2014-07-01 17:01:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3039: read oid 155 snap -1 2014-07-01 17:01:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3040: read oid 171 snap -1 2014-07-01 17:01:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 229 current snap is 0 2014-07-01 17:01:13,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: seq_num 1900 ranges {8364032=434176,8798208=401408} 2014-07-01 17:01:13,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: writing mira0435601-229 from 8364032 to 8798208 tid 1 2014-07-01 17:01:13,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: writing mira0435601-229 from 8798208 to 9199616 tid 2 2014-07-01 17:01:13,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 30 current snap is 0 2014-07-01 17:01:13,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: seq_num 1901 ranges {2195456=462848,2658304=659456,3317760=712704,4030464=233472} 2014-07-01 17:01:13,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0435601-30 from 2195456 to 2658304 tid 1 2014-07-01 17:01:13,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1641: done (3 left) 2014-07-01 17:01:13,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1642: done (2 left) 2014-07-01 17:01:13,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:1643: done (1 left) 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1644: done (0 left) 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: write oid 101 current snap is 164 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: seq_num 918 ranges {557542=712770,1357541=1} 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: writing mira0437250-101 from 557542 to 1270312 tid 1 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: writing mira0437250-101 from 1357541 to 1357542 tid 2 2014-07-01 17:01:13,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:1646: delete oid 61 current snap is 164 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1646: done (1 left) 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1647: snap_create 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 1 to mira0437250-101 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 2 to mira0437250-101 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 4 to mira0437250-101 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: finishing write tid 5 to mira0437250-101 2014-07-01 17:01:13,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 644 (ObjNum 918 snap 164 seq_num 918) dirty exists 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0435601-30 from 2658304 to 3317760 tid 2 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0435601-30 from 3317760 to 4030464 tid 3 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0435601-30 from 4030464 to 4263936 tid 4 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3043: read oid 22 snap -1 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3044: read oid 480 snap -1 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: finishing write tid 1 to mira0435601-443 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: finishing write tid 2 to mira0435601-443 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: finishing write tid 4 to mira0435601-443 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: finishing write tid 5 to mira0435601-443 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 289 (ObjNum 1898 snap 0 seq_num 1898) dirty exists 2014-07-01 17:01:13,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3040: expect (ObjNum 1288 snap 0 seq_num 1288) 2014-07-01 17:01:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1645: done (1 left) 2014-07-01 17:01:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1647: done (0 left) 2014-07-01 17:01:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: read oid 122 snap -1 2014-07-01 17:01:14,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1649: read oid 156 snap -1 2014-07-01 17:01:14,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1650: delete oid 500 current snap is 165 2014-07-01 17:01:14,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1649: done (2 left) 2014-07-01 17:01:14,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1650: done (1 left) 2014-07-01 17:01:14,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: write oid 498 current snap is 165 2014-07-01 17:01:15,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: seq_num 921 ranges {658784=664279,1830961=721749,3058783=1} 2014-07-01 17:01:15,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: writing mira0437250-498 from 658784 to 1323063 tid 1 2014-07-01 17:01:15,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: writing mira0437250-498 from 1830961 to 2552710 tid 2 2014-07-01 17:01:15,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: writing mira0437250-498 from 3058783 to 3058784 tid 3 2014-07-01 17:01:15,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: delete oid 162 current snap is 165 2014-07-01 17:01:15,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: expect (ObjNum 121 snap 0 seq_num 121) 2014-07-01 17:01:15,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: finishing write tid 1 to mira0435601-229 2014-07-01 17:01:15,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: finishing write tid 2 to mira0435601-229 2014-07-01 17:01:15,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: expect (ObjNum 1739 snap 0 seq_num 1739) 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: finishing write tid 1 to mira0437250-498 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: finishing write tid 2 to mira0437250-498 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: finishing write tid 3 to mira0437250-498 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: finishing write tid 5 to mira0437250-498 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: finishing write tid 6 to mira0437250-498 2014-07-01 17:01:16,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 564 (ObjNum 921 snap 165 seq_num 921) dirty exists 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1648: done (2 left) 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1651: done (1 left) 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1652: done (0 left) 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: rollback oid 463 current snap is 165 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 463 to 80 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: rollback oid 356 current snap is 165 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 356 to 163 2014-07-01 17:01:16,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1655: snap_create 2014-07-01 17:01:16,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 356 v 760 (ObjNum 355 snap 0 seq_num 355) dirty exists 2014-07-01 17:01:16,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 883 (ObjNum 616 snap 50 seq_num 616) dirty exists 2014-07-01 17:01:16,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: finishing write tid 4 to mira0435601-229 2014-07-01 17:01:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: finishing write tid 5 to mira0435601-229 2014-07-01 17:01:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 271 (ObjNum 1900 snap 0 seq_num 1900) dirty exists 2014-07-01 17:01:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 1 to mira0435601-30 2014-07-01 17:01:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 2 to mira0435601-30 2014-07-01 17:01:16,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3044: expect (ObjNum 1588 snap 0 seq_num 1588) 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 3 to mira0435601-30 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 4 to mira0435601-30 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 6 to mira0435601-30 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 7 to mira0435601-30 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 160 (ObjNum 1901 snap 0 seq_num 1901) dirty exists 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3043: expect (ObjNum 1670 snap 0 seq_num 1670) 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1653: done (2 left) 2014-07-01 17:01:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:1654: done (1 left) 2014-07-01 17:01:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1655: done (0 left) 2014-07-01 17:01:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: delete oid 268 current snap is 166 2014-07-01 17:01:16,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:1656: done (0 left) 2014-07-01 17:01:16,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: read oid 306 snap -1 2014-07-01 17:01:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: delete oid 393 current snap is 166 2014-07-01 17:01:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1658: done (1 left) 2014-07-01 17:01:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: write oid 451 current snap is 166 2014-07-01 17:01:17,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: seq_num 925 ranges {455254=1} 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: writing mira0437250-451 from 455254 to 455255 tid 1 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1660: snap_create 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: finishing write tid 1 to mira0437250-451 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: finishing write tid 3 to mira0437250-451 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: finishing write tid 4 to mira0437250-451 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 1014 (ObjNum 925 snap 166 seq_num 925) dirty exists 2014-07-01 17:01:17,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: expect (ObjNum 305 snap 0 seq_num 305) 2014-07-01 17:01:17,905.905 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:17,905.905 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:17,906.906 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:01:17,906.906 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:01:17,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:1657: done (2 left) 2014-07-01 17:01:17,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1659: done (1 left) 2014-07-01 17:01:17,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1660: done (0 left) 2014-07-01 17:01:17,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: delete oid 375 current snap is 167 2014-07-01 17:01:17,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:1661: done (0 left) 2014-07-01 17:01:17,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:1662: rollback oid 119 current snap is 167 2014-07-01 17:01:17,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 119 to 136 2014-07-01 17:01:17,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:1663: snap_remove snap 112 2014-07-01 17:01:17,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 659 (ObjNum 118 snap 0 seq_num 118) dirty exists 2014-07-01 17:01:18,663.663 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:01:18,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1662: done (1 left) 2014-07-01 17:01:18,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:1663: done (0 left) 2014-07-01 17:01:18,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1664: rollback oid 395 current snap is 167 2014-07-01 17:01:18,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 395 to 165 2014-07-01 17:01:18,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: write oid 75 current snap is 167 2014-07-01 17:01:18,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: seq_num 927 ranges {743890=567297,1943889=1} 2014-07-01 17:01:18,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: writing mira0437250-75 from 743890 to 1311187 tid 1 2014-07-01 17:01:18,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: writing mira0437250-75 from 1943889 to 1943890 tid 2 2014-07-01 17:01:19,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:1666: delete oid 473 current snap is 167 2014-07-01 17:01:19,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 884 (ObjNum 394 snap 0 seq_num 394) dirty exists 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1664: done (2 left) 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1666: done (1 left) 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1667: snap_remove snap 166 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: finishing write tid 1 to mira0437250-75 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: finishing write tid 2 to mira0437250-75 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: finishing write tid 4 to mira0437250-75 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: finishing write tid 5 to mira0437250-75 2014-07-01 17:01:19,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 548 (ObjNum 927 snap 167 seq_num 927) dirty exists 2014-07-01 17:01:19,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3039: expect (ObjNum 1827 snap 0 seq_num 1827) 2014-07-01 17:01:19,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:1665: done (1 left) 2014-07-01 17:01:19,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:1667: done (0 left) 2014-07-01 17:01:19,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:1668: delete oid 61 current snap is 167 2014-07-01 17:01:19,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1668: done (0 left) 2014-07-01 17:01:19,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: read oid 262 snap -1 2014-07-01 17:01:19,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: delete oid 218 current snap is 167 2014-07-01 17:01:19,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: expect (ObjNum 261 snap 0 seq_num 261) 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1669: done (1 left) 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1670: done (0 left) 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: write oid 241 current snap is 167 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: seq_num 931 ranges {539337=758387,1876475=561621,2994602=344735} 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: writing mira0437250-241 from 539337 to 1297724 tid 1 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: writing mira0437250-241 from 1876475 to 2438096 tid 2 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: writing mira0437250-241 from 2994602 to 3339337 tid 3 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: read oid 112 snap -1 2014-07-01 17:01:20,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 1 to mira0437250-241 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: write oid 469 current snap is 167 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: seq_num 932 ranges {84166=1} 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: writing mira0437250-469 from 84166 to 84167 tid 1 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: rollback oid 453 current snap is 167 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 453 to 163 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1675: snap_remove snap 145 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 2 to mira0437250-241 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 453 v 582 (ObjNum 711 snap 95 seq_num 711) dirty exists 2014-07-01 17:01:20,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: finishing write tid 1 to mira0437250-469 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: finishing write tid 3 to mira0437250-469 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 3 to mira0437250-241 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: finishing write tid 4 to mira0437250-469 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 694 (ObjNum 932 snap 167 seq_num 932) dirty exists 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 5 to mira0437250-241 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: finishing write tid 6 to mira0437250-241 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 764 (ObjNum 931 snap 167 seq_num 931) dirty exists 2014-07-01 17:01:20,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: expect (ObjNum 111 snap 0 seq_num 111) 2014-07-01 17:01:20,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: done (7 left) 2014-07-01 17:01:20,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: done (6 left) 2014-07-01 17:01:20,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3039: done (5 left) 2014-07-01 17:01:20,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3040: done (4 left) 2014-07-01 17:01:20,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: done (3 left) 2014-07-01 17:01:20,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: done (2 left) 2014-07-01 17:01:20,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3043: done (1 left) 2014-07-01 17:01:20,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3044: done (0 left) 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 257 current snap is 0 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: seq_num 1902 ranges {4653056=733184,5386240=487424,5873664=716800} 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: writing mira0435601-257 from 4653056 to 5386240 tid 1 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: writing mira0435601-257 from 5386240 to 5873664 tid 2 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: writing mira0435601-257 from 5873664 to 6590464 tid 3 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3046: read oid 222 snap -1 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: finishing write tid 1 to mira0435601-257 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: finishing write tid 2 to mira0435601-257 2014-07-01 17:01:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3047: read oid 152 snap -1 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: seq_num 1903 ranges {225280=253952} 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: writing mira0435601-186 from 225280 to 479232 tid 1 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3049: read oid 14 snap -1 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 454 current snap is 0 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: seq_num 1904 ranges {1720320=684032,2404352=651264,3055616=151552} 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0435601-454 from 1720320 to 2404352 tid 1 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0435601-454 from 2404352 to 3055616 tid 2 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0435601-454 from 3055616 to 3207168 tid 3 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: delete oid 307 current snap is 0 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: finishing write tid 3 to mira0435601-257 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: finishing write tid 5 to mira0435601-257 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: finishing write tid 1 to mira0435601-186 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: finishing write tid 3 to mira0435601-186 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: finishing write tid 6 to mira0435601-257 2014-07-01 17:01:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 257 v 173 (ObjNum 1902 snap 0 seq_num 1902) dirty exists 2014-07-01 17:01:20,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:3046: expect (ObjNum 1455 snap 0 seq_num 1455) 2014-07-01 17:01:20,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1671: done (4 left) 2014-07-01 17:01:20,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1672: done (3 left) 2014-07-01 17:01:20,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:1673: done (2 left) 2014-07-01 17:01:20,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:1674: done (1 left) 2014-07-01 17:01:20,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:1675: done (0 left) 2014-07-01 17:01:20,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: rollback oid 220 current snap is 167 2014-07-01 17:01:20,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 220 to 120 2014-07-01 17:01:20,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:1677: snap_remove snap 161 2014-07-01 17:01:21,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 519 (ObjNum 725 snap 97 seq_num 725) dirty exists 2014-07-01 17:01:21,704.704 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:01:21,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1676: done (1 left) 2014-07-01 17:01:21,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1677: done (0 left) 2014-07-01 17:01:21,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: read oid 149 snap -1 2014-07-01 17:01:21,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: delete oid 378 current snap is 167 2014-07-01 17:01:22,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: expect (ObjNum 523 snap 12 seq_num 523) 2014-07-01 17:01:22,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1678: done (1 left) 2014-07-01 17:01:22,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1679: done (0 left) 2014-07-01 17:01:22,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: delete oid 412 current snap is 167 2014-07-01 17:01:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1680: done (0 left) 2014-07-01 17:01:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: read oid 37 snap 162 2014-07-01 17:01:22,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1682: delete oid 387 current snap is 167 2014-07-01 17:01:22,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1682: done (1 left) 2014-07-01 17:01:22,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: snap_remove snap 80 2014-07-01 17:01:22,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: expect (ObjNum 887 snap 151 seq_num 887) 2014-07-01 17:01:22,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1681: done (1 left) 2014-07-01 17:01:22,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1683: done (0 left) 2014-07-01 17:01:22,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: rollback oid 372 current snap is 167 2014-07-01 17:01:22,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 372 to 152 2014-07-01 17:01:22,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: read oid 76 snap -1 2014-07-01 17:01:22,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: rollback oid 485 current snap is 167 2014-07-01 17:01:22,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 485 to 120 2014-07-01 17:01:22,996.996 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:01:23,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1687: snap_create 2014-07-01 17:01:23,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: expect (ObjNum 75 snap 0 seq_num 75) 2014-07-01 17:01:23,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 838 (ObjNum 484 snap 0 seq_num 484) dirty exists 2014-07-01 17:01:23,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 481 (ObjNum 371 snap 0 seq_num 371) dirty exists 2014-07-01 17:01:23,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 1 to mira0435601-454 2014-07-01 17:01:23,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 2 to mira0435601-454 2014-07-01 17:01:23,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 3 to mira0435601-454 2014-07-01 17:01:23,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3047: expect (ObjNum 1740 snap 0 seq_num 1740) 2014-07-01 17:01:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1684: done (3 left) 2014-07-01 17:01:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1685: done (2 left) 2014-07-01 17:01:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1686: done (1 left) 2014-07-01 17:01:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1687: done (0 left) 2014-07-01 17:01:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1688: rollback oid 432 current snap is 168 2014-07-01 17:01:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 432 to 164 2014-07-01 17:01:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: read oid 362 snap -1 2014-07-01 17:01:24,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: write oid 67 current snap is 168 2014-07-01 17:01:24,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: seq_num 936 ranges {680638=689600,1480637=1} 2014-07-01 17:01:24,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: writing mira0437250-67 from 680638 to 1370238 tid 1 2014-07-01 17:01:24,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: writing mira0437250-67 from 1480637 to 1480638 tid 2 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: snap_remove snap 146 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 432 v 692 (ObjNum 431 snap 0 seq_num 431) dirty exists 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: expect (ObjNum 361 snap 0 seq_num 361) 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 1 to mira0437250-67 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 2 to mira0437250-67 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 4 to mira0437250-67 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: finishing write tid 5 to mira0437250-67 2014-07-01 17:01:24,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 448 (ObjNum 936 snap 168 seq_num 936) dirty exists 2014-07-01 17:01:25,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1688: done (3 left) 2014-07-01 17:01:25,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1689: done (2 left) 2014-07-01 17:01:25,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1690: done (1 left) 2014-07-01 17:01:25,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1691: done (0 left) 2014-07-01 17:01:25,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:1692: snap_create 2014-07-01 17:01:25,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3049: expect (ObjNum 1569 snap 0 seq_num 1569) 2014-07-01 17:01:26,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1692: done (0 left) 2014-07-01 17:01:26,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: write oid 69 current snap is 169 2014-07-01 17:01:26,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: seq_num 937 ranges {625467=661405,2070899=154568} 2014-07-01 17:01:26,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: writing mira0437250-69 from 625467 to 1286872 tid 1 2014-07-01 17:01:26,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: writing mira0437250-69 from 2070899 to 2225467 tid 2 2014-07-01 17:01:26,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: snap_create 2014-07-01 17:01:26,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 1 to mira0437250-69 2014-07-01 17:01:26,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 2 to mira0437250-69 2014-07-01 17:01:26,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 4 to mira0437250-69 2014-07-01 17:01:26,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: finishing write tid 5 to mira0437250-69 2014-07-01 17:01:26,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 69 v 915 (ObjNum 937 snap 169 seq_num 937) dirty exists 2014-07-01 17:01:27,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:1693: done (1 left) 2014-07-01 17:01:27,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:1694: done (0 left) 2014-07-01 17:01:27,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: snap_create 2014-07-01 17:01:28,013.013 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:28,013.013 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:28,013.013 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 1 to 0.468100274209 2014-07-01 17:01:28,014.014 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 0.468100274209' 2014-07-01 17:01:28,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:1695: done (0 left) 2014-07-01 17:01:28,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: write oid 198 current snap is 171 2014-07-01 17:01:28,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: seq_num 938 ranges {685625=785657,2017735=589883,3301354=184271} 2014-07-01 17:01:28,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0437250-198 from 685625 to 1471282 tid 1 2014-07-01 17:01:28,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0437250-198 from 2017735 to 2607618 tid 2 2014-07-01 17:01:28,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: writing mira0437250-198 from 3301354 to 3485625 tid 3 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: read oid 142 snap -1 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: read oid 114 snap -1 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: read oid 419 snap 120 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: read oid 413 snap -1 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1701: snap_remove snap 162 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 1 to mira0437250-198 2014-07-01 17:01:28,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 17:01:28,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 2 to mira0437250-198 2014-07-01 17:01:28,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: expect (ObjNum 418 snap 0 seq_num 418) 2014-07-01 17:01:28,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 3 to mira0437250-198 2014-07-01 17:01:28,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 5 to mira0437250-198 2014-07-01 17:01:28,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: finishing write tid 6 to mira0437250-198 2014-07-01 17:01:28,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 599 (ObjNum 938 snap 171 seq_num 938) dirty exists 2014-07-01 17:01:28,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: expect (ObjNum 113 snap 0 seq_num 113) 2014-07-01 17:01:29,104.104 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 0.4681 (8306772) 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 5 to mira0435601-454 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 6 to mira0435601-454 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 179 (ObjNum 1904 snap 0 seq_num 1904) dirty exists 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: finishing write tid 4 to mira0435601-186 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 175 (ObjNum 1903 snap 0 seq_num 1903) dirty exists 2014-07-01 17:01:29,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: done (6 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3046: done (5 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3047: done (4 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: done (3 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3049: done (2 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: done (1 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: done (0 left) 2014-07-01 17:01:29,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: read oid 145 snap -1 2014-07-01 17:01:29,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:3053: read oid 430 snap -1 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 247 current snap is 0 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: seq_num 1906 ranges {4059136=425984,4485120=466944,4952064=335872} 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: writing mira0435601-247 from 4059136 to 4485120 tid 1 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: writing mira0435601-247 from 4485120 to 4952064 tid 2 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: writing mira0435601-247 from 4952064 to 5287936 tid 3 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: read oid 298 snap -1 2014-07-01 17:01:29,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3053: expect (ObjNum 1587 snap 0 seq_num 1587) 2014-07-01 17:01:29,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: expect (ObjNum 412 snap 0 seq_num 412) 2014-07-01 17:01:29,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1696: done (5 left) 2014-07-01 17:01:29,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1697: done (4 left) 2014-07-01 17:01:29,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1698: done (3 left) 2014-07-01 17:01:29,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1699: done (2 left) 2014-07-01 17:01:29,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:1700: done (1 left) 2014-07-01 17:01:29,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1701: done (0 left) 2014-07-01 17:01:29,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: rollback oid 462 current snap is 171 2014-07-01 17:01:29,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 462 to 143 2014-07-01 17:01:29,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: rollback oid 430 current snap is 171 2014-07-01 17:01:29,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 430 to 120 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1704: snap_remove snap 170 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 697 (ObjNum 461 snap 0 seq_num 461) dirty exists 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 637 (ObjNum 429 snap 0 seq_num 429) dirty exists 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1702: done (2 left) 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1703: done (1 left) 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1704: done (0 left) 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: write oid 143 current snap is 171 2014-07-01 17:01:30,127.127 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: seq_num 939 ranges {780855=757461,1979419=660078,3082355=98500} 2014-07-01 17:01:30,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: writing mira0437250-143 from 780855 to 1538316 tid 1 2014-07-01 17:01:30,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: writing mira0437250-143 from 1979419 to 2639497 tid 2 2014-07-01 17:01:30,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: writing mira0437250-143 from 3082355 to 3180855 tid 3 2014-07-01 17:01:30,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: read oid 240 snap -1 2014-07-01 17:01:30,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1707: snap_remove snap 143 2014-07-01 17:01:30,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: expect (ObjNum 239 snap 0 seq_num 239) 2014-07-01 17:01:30,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 1 to mira0437250-143 2014-07-01 17:01:30,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 2 to mira0437250-143 2014-07-01 17:01:30,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 3 to mira0437250-143 2014-07-01 17:01:30,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 5 to mira0437250-143 2014-07-01 17:01:30,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: finishing write tid 6 to mira0437250-143 2014-07-01 17:01:30,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 491 (ObjNum 939 snap 171 seq_num 939) dirty exists 2014-07-01 17:01:31,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1705: done (2 left) 2014-07-01 17:01:31,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:1706: done (1 left) 2014-07-01 17:01:31,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1707: done (0 left) 2014-07-01 17:01:31,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:1708: snap_remove snap 120 2014-07-01 17:01:32,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1708: done (0 left) 2014-07-01 17:01:32,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: read oid 371 snap 160 2014-07-01 17:01:32,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:1710: delete oid 239 current snap is 171 2014-07-01 17:01:32,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: expect (ObjNum 370 snap 0 seq_num 370) 2014-07-01 17:01:32,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1709: done (1 left) 2014-07-01 17:01:32,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1710: done (0 left) 2014-07-01 17:01:32,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1711: snap_create 2014-07-01 17:01:32,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: expect (ObjNum 1845 snap 0 seq_num 1845) 2014-07-01 17:01:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1711: done (0 left) 2014-07-01 17:01:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: snap_remove snap 51 2014-07-01 17:01:34,118.118 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:34,119.119 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:34,119.119 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:01:34,119.119 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:01:34,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1712: done (0 left) 2014-07-01 17:01:34,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:1713: snap_remove snap 165 2014-07-01 17:01:34,946.946 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: finishing write tid 1 to mira0435601-247 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: finishing write tid 2 to mira0435601-247 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: finishing write tid 3 to mira0435601-247 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: finishing write tid 5 to mira0435601-247 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: finishing write tid 6 to mira0435601-247 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 283 (ObjNum 1906 snap 0 seq_num 1906) dirty exists 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: done (3 left) 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3053: done (2 left) 2014-07-01 17:01:35,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: done (1 left) 2014-07-01 17:01:35,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: done (0 left) 2014-07-01 17:01:35,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 395 current snap is 0 2014-07-01 17:01:35,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: seq_num 1907 ranges {2809856=503808,3313664=401408} 2014-07-01 17:01:35,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: writing mira0435601-395 from 2809856 to 3313664 tid 1 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: writing mira0435601-395 from 3313664 to 3715072 tid 2 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3057: read oid 464 snap -1 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3058: read oid 412 snap -1 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3059: read oid 69 snap -1 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3060: read oid 375 snap -1 2014-07-01 17:01:35,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 266 current snap is 0 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: seq_num 1908 ranges {2129920=774144} 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: writing mira0435601-266 from 2129920 to 2904064 tid 1 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 15 current snap is 0 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: seq_num 1909 ranges {5226496=647168} 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: writing mira0435601-15 from 5226496 to 5873664 tid 1 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 470 current snap is 0 2014-07-01 17:01:35,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: seq_num 1910 ranges {3522560=675840,4198400=692224,4890624=434176,5324800=77824} 2014-07-01 17:01:35,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: writing mira0435601-470 from 3522560 to 4198400 tid 1 2014-07-01 17:01:35,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: writing mira0435601-470 from 4198400 to 4890624 tid 2 2014-07-01 17:01:35,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: writing mira0435601-470 from 4890624 to 5324800 tid 3 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: writing mira0435601-470 from 5324800 to 5402624 tid 4 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 457 current snap is 0 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: seq_num 1911 ranges {3973120=544768,4517888=679936,5197824=524288} 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: writing mira0435601-457 from 3973120 to 4517888 tid 1 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: writing mira0435601-457 from 4517888 to 5197824 tid 2 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: writing mira0435601-457 from 5197824 to 5722112 tid 3 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3065: delete oid 117 current snap is 0 2014-07-01 17:01:35,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3059: expect (ObjNum 68 snap 0 seq_num 68) 2014-07-01 17:01:35,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1713: done (0 left) 2014-07-01 17:01:35,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:1714: read oid 148 snap -1 2014-07-01 17:01:35,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1715: rollback oid 376 current snap is 172 2014-07-01 17:01:35,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 376 to 159 2014-07-01 17:01:35,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: snap_remove snap 164 2014-07-01 17:01:35,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 0 (ObjNum 3 snap 0 seq_num 0) dirty dne 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 1 to mira0435601-395 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: finishing write tid 1 to mira0435601-15 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: finishing write tid 3 to mira0435601-15 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 2 to mira0435601-395 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: finishing write tid 4 to mira0435601-15 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 159 (ObjNum 1909 snap 0 seq_num 1909) dirty exists 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 1 to mira0435601-470 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3059: done (9 left) 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3060: done (8 left) 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: done (7 left) 2014-07-01 17:01:36,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3065: done (6 left) 2014-07-01 17:01:36,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:3066: read oid 163 snap -1 2014-07-01 17:01:36,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:3058: expect (ObjNum 1053 snap 0 seq_num 1053) 2014-07-01 17:01:36,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:1714: done (2 left) 2014-07-01 17:01:36,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1715: done (1 left) 2014-07-01 17:01:36,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1716: done (0 left) 2014-07-01 17:01:36,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: snap_create 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 4 to mira0435601-395 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 2 to mira0435601-470 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 3 to mira0435601-470 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 4 to mira0435601-470 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 6 to mira0435601-470 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: finishing write tid 1 to mira0435601-266 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: finishing write tid 3 to mira0435601-266 2014-07-01 17:01:37,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: finishing write tid 1 to mira0435601-457 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: finishing write tid 2 to mira0435601-457 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: finishing write tid 3 to mira0435601-457 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3058: done (6 left) 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3067: read oid 284 snap -1 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: finishing write tid 5 to mira0435601-457 2014-07-01 17:01:37,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 5 to mira0435601-395 2014-07-01 17:01:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 263 (ObjNum 1907 snap 0 seq_num 1907) dirty exists 2014-07-01 17:01:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: finishing write tid 6 to mira0435601-457 2014-07-01 17:01:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 457 v 229 (ObjNum 1911 snap 0 seq_num 1911) dirty exists 2014-07-01 17:01:37,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: finishing write tid 7 to mira0435601-470 2014-07-01 17:01:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 247 (ObjNum 1910 snap 0 seq_num 1910) dirty exists 2014-07-01 17:01:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: finishing write tid 4 to mira0435601-266 2014-07-01 17:01:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 216 (ObjNum 1908 snap 0 seq_num 1908) dirty exists 2014-07-01 17:01:37,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3057: expect (ObjNum 1530 snap 0 seq_num 1530) 2014-07-01 17:01:37,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1717: done (0 left) 2014-07-01 17:01:37,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: delete oid 470 current snap is 173 2014-07-01 17:01:37,993.993 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 34' 2014-07-01 17:01:38,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:1718: done (0 left) 2014-07-01 17:01:38,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:1719: read oid 146 snap -1 2014-07-01 17:01:38,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:1720: delete oid 207 current snap is 173 2014-07-01 17:01:38,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1720: done (1 left) 2014-07-01 17:01:38,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1721: read oid 71 snap -1 2014-07-01 17:01:38,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:1722: snap_remove snap 136 2014-07-01 17:01:38,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:1719: expect (ObjNum 582 snap 31 seq_num 582) 2014-07-01 17:01:39,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1719: done (2 left) 2014-07-01 17:01:39,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:1721: done (1 left) 2014-07-01 17:01:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1722: done (0 left) 2014-07-01 17:01:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1723: rollback oid 433 current snap is 173 2014-07-01 17:01:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 433 to 159 2014-07-01 17:01:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: write oid 189 current snap is 173 2014-07-01 17:01:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: seq_num 943 ranges {425902=432199,1412333=213569} 2014-07-01 17:01:39,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: writing mira0437250-189 from 425902 to 858101 tid 1 2014-07-01 17:01:39,180.180 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 34 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: writing mira0437250-189 from 1412333 to 1625902 tid 2 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1725: delete oid 321 current snap is 173 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1725: done (2 left) 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1726: snap_remove snap 152 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: finishing write tid 1 to mira0437250-189 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: finishing write tid 2 to mira0437250-189 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: finishing write tid 4 to mira0437250-189 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 595 (ObjNum 432 snap 0 seq_num 432) dirty exists 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: finishing write tid 5 to mira0437250-189 2014-07-01 17:01:39,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 612 (ObjNum 943 snap 173 seq_num 943) dirty exists 2014-07-01 17:01:40,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1723: done (2 left) 2014-07-01 17:01:40,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1724: done (1 left) 2014-07-01 17:01:40,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:1726: done (0 left) 2014-07-01 17:01:40,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:1727: snap_remove snap 160 2014-07-01 17:01:41,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1727: done (0 left) 2014-07-01 17:01:41,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: rollback oid 349 current snap is 173 2014-07-01 17:01:41,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 349 to 169 2014-07-01 17:01:41,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: write oid 14 current snap is 173 2014-07-01 17:01:41,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: seq_num 945 ranges {466985=411903,1525268=541717} 2014-07-01 17:01:41,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: writing mira0437250-14 from 466985 to 878888 tid 1 2014-07-01 17:01:41,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: writing mira0437250-14 from 1525268 to 2066985 tid 2 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: write oid 380 current snap is 173 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: seq_num 946 ranges {562215=783708,1762214=1} 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: writing mira0437250-380 from 562215 to 1345923 tid 1 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: writing mira0437250-380 from 1762214 to 1762215 tid 2 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: write oid 386 current snap is 173 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: seq_num 947 ranges {70948=1} 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: writing mira0437250-386 from 70948 to 70949 tid 1 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1732: read oid 24 snap -1 2014-07-01 17:01:41,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: finishing write tid 1 to mira0437250-14 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 1035 (ObjNum 697 snap 90 seq_num 697) dirty exists 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1728: done (4 left) 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: write oid 469 current snap is 173 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: seq_num 948 ranges {566178=763412,1795472=429755,2972360=758819,3766177=1} 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0437250-469 from 566178 to 1329590 tid 1 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0437250-469 from 1795472 to 2225227 tid 2 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0437250-469 from 2972360 to 3731179 tid 3 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: writing mira0437250-469 from 3766177 to 3766178 tid 4 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1734: delete oid 232 current snap is 173 2014-07-01 17:01:41,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: finishing write tid 2 to mira0437250-14 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: finishing write tid 4 to mira0437250-14 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: finishing write tid 5 to mira0437250-14 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 478 (ObjNum 945 snap 173 seq_num 945) dirty exists 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: finishing write tid 1 to mira0437250-386 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: finishing write tid 3 to mira0437250-386 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: finishing write tid 1 to mira0437250-380 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: finishing write tid 2 to mira0437250-380 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: finishing write tid 4 to mira0437250-386 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 616 (ObjNum 947 snap 173 seq_num 947) dirty exists 2014-07-01 17:01:41,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:1732: expect (ObjNum 23 snap 0 seq_num 23) 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: finishing write tid 4 to mira0437250-380 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: finishing write tid 5 to mira0437250-380 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 380 v 714 (ObjNum 946 snap 173 seq_num 946) dirty exists 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 1 to mira0437250-469 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 2 to mira0437250-469 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 3 to mira0437250-469 2014-07-01 17:01:42,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 4 to mira0437250-469 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 6 to mira0437250-469 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: finishing write tid 7 to mira0437250-469 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 760 (ObjNum 948 snap 173 seq_num 948) dirty exists 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1729: done (5 left) 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1730: done (4 left) 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1731: done (3 left) 2014-07-01 17:01:42,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:1732: done (2 left) 2014-07-01 17:01:42,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:1733: done (1 left) 2014-07-01 17:01:42,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:1734: done (0 left) 2014-07-01 17:01:42,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: write oid 394 current snap is 173 2014-07-01 17:01:42,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: seq_num 950 ranges {455838=458445,1255837=1} 2014-07-01 17:01:42,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: writing mira0437250-394 from 455838 to 914283 tid 1 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: writing mira0437250-394 from 1255837 to 1255838 tid 2 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: write oid 41 current snap is 173 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: seq_num 951 ranges {551068=400000} 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: writing mira0437250-41 from 551068 to 951068 tid 1 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1737: rollback oid 27 current snap is 173 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 155 2014-07-01 17:01:42,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: read oid 289 snap -1 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: write oid 12 current snap is 173 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: seq_num 952 ranges {763673=400000} 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: writing mira0437250-12 from 763673 to 1163673 tid 1 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1740: snap_remove snap 147 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: finishing write tid 1 to mira0437250-394 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: finishing write tid 2 to mira0437250-394 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: finishing write tid 4 to mira0437250-394 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 716 (ObjNum 647 snap 67 seq_num 647) dirty exists 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: expect (ObjNum 859 snap 145 seq_num 859) 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: finishing write tid 1 to mira0437250-41 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: finishing write tid 3 to mira0437250-41 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: finishing write tid 5 to mira0437250-394 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 987 (ObjNum 950 snap 173 seq_num 950) dirty exists 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: finishing write tid 4 to mira0437250-41 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 703 (ObjNum 951 snap 173 seq_num 951) dirty exists 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 1 to mira0437250-12 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 3 to mira0437250-12 2014-07-01 17:01:42,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: finishing write tid 4 to mira0437250-12 2014-07-01 17:01:42,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 641 (ObjNum 952 snap 173 seq_num 952) dirty exists 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1735: done (5 left) 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1736: done (4 left) 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1737: done (3 left) 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1738: done (2 left) 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1739: done (1 left) 2014-07-01 17:01:43,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1740: done (0 left) 2014-07-01 17:01:43,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:1741: snap_create 2014-07-01 17:01:44,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1741: done (0 left) 2014-07-01 17:01:44,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: snap_create 2014-07-01 17:01:44,193.193 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:44,193.193 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:44,194.194 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:01:44,194.194 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:01:44,194.194 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:01:44,194.194 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:01:44,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3066: expect (ObjNum 1205 snap 0 seq_num 1205) 2014-07-01 17:01:45,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1742: done (0 left) 2014-07-01 17:01:45,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: delete oid 233 current snap is 175 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1743: done (0 left) 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: read oid 273 snap -1 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: rollback oid 419 current snap is 175 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 419 to 140 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: rollback oid 287 current snap is 175 2014-07-01 17:01:45,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 287 to 149 2014-07-01 17:01:45,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: write oid 340 current snap is 175 2014-07-01 17:01:45,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: seq_num 954 ranges {652309=533182,1856649=503150,2652308=1} 2014-07-01 17:01:45,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: writing mira0437250-340 from 652309 to 1185491 tid 1 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: writing mira0437250-340 from 1856649 to 2359799 tid 2 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: writing mira0437250-340 from 2652308 to 2652309 tid 3 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 736 (ObjNum 418 snap 0 seq_num 418) dirty exists 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1745: done (3 left) 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: write oid 57 current snap is 175 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: seq_num 955 ranges {747539=506011,1819358=528181} 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: writing mira0437250-57 from 747539 to 1253550 tid 1 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: writing mira0437250-57 from 1819358 to 2347539 tid 2 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1749: rollback oid 108 current snap is 175 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 108 to 171 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 630 (ObjNum 633 snap 60 seq_num 633) dirty exists 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: expect (ObjNum 272 snap 0 seq_num 272) 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: finishing write tid 1 to mira0437250-340 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: finishing write tid 2 to mira0437250-340 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: finishing write tid 3 to mira0437250-340 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: finishing write tid 5 to mira0437250-340 2014-07-01 17:01:45,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: finishing write tid 6 to mira0437250-340 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 652 (ObjNum 954 snap 175 seq_num 954) dirty exists 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 726 (ObjNum 107 snap 0 seq_num 107) dirty exists 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: finishing write tid 1 to mira0437250-57 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: finishing write tid 2 to mira0437250-57 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: finishing write tid 4 to mira0437250-57 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: finishing write tid 5 to mira0437250-57 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 1109 (ObjNum 955 snap 175 seq_num 955) dirty exists 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1744: done (4 left) 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1746: done (3 left) 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1747: done (2 left) 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1748: done (1 left) 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1749: done (0 left) 2014-07-01 17:01:45,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:1750: snap_remove snap 172 2014-07-01 17:01:46,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:1750: done (0 left) 2014-07-01 17:01:46,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: snap_remove snap 127 2014-07-01 17:01:47,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3067: expect (ObjNum 1672 snap 0 seq_num 1672) 2014-07-01 17:01:47,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1751: done (0 left) 2014-07-01 17:01:47,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1752: delete oid 140 current snap is 175 2014-07-01 17:01:47,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1752: done (0 left) 2014-07-01 17:01:47,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: read oid 198 snap -1 2014-07-01 17:01:47,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: snap_remove snap 159 2014-07-01 17:01:47,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: expect (ObjNum 938 snap 171 seq_num 938) 2014-07-01 17:01:48,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:1753: done (1 left) 2014-07-01 17:01:48,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1754: done (0 left) 2014-07-01 17:01:48,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1755: rollback oid 255 current snap is 175 2014-07-01 17:01:48,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 255 to 173 2014-07-01 17:01:48,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: read oid 284 snap 168 2014-07-01 17:01:48,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: write oid 476 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: seq_num 957 ranges {352525=1} 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: writing mira0437250-476 from 352525 to 352526 tid 1 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1758: rollback oid 71 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 71 to 156 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1759: rollback oid 360 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 360 to 169 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: write oid 62 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: seq_num 958 ranges {47755=1} 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: writing mira0437250-62 from 47755 to 47756 tid 1 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: write oid 383 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: seq_num 959 ranges {792585=1} 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: writing mira0437250-383 from 792585 to 792586 tid 1 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: read oid 146 snap -1 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 660 (ObjNum 70 snap 0 seq_num 70) dirty exists 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1758: done (7 left) 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: rollback oid 378 current snap is 175 2014-07-01 17:01:48,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 378 to 155 2014-07-01 17:01:48,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1764: read oid 292 snap -1 2014-07-01 17:01:48,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: read oid 241 snap -1 2014-07-01 17:01:48,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1766: snap_remove snap 171 2014-07-01 17:01:48,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1764: expect (ObjNum 876 snap 147 seq_num 876) 2014-07-01 17:01:48,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: expect (ObjNum 781 snap 124 seq_num 781) 2014-07-01 17:01:48,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: done (6 left) 2014-07-01 17:01:48,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3057: done (5 left) 2014-07-01 17:01:48,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: done (4 left) 2014-07-01 17:01:48,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: done (3 left) 2014-07-01 17:01:48,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: done (2 left) 2014-07-01 17:01:48,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3066: done (1 left) 2014-07-01 17:01:48,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3067: done (0 left) 2014-07-01 17:01:48,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 421 current snap is 0 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: seq_num 1913 ranges {1413120=491520,1904640=442368,2347008=360448} 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: writing mira0435601-421 from 1413120 to 1904640 tid 1 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: writing mira0435601-421 from 1904640 to 2347008 tid 2 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: writing mira0435601-421 from 2347008 to 2707456 tid 3 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 348 current snap is 0 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: seq_num 1914 ranges {1515520=765952,2281472=401408} 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: writing mira0435601-348 from 1515520 to 2281472 tid 1 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: writing mira0435601-348 from 2281472 to 2682880 tid 2 2014-07-01 17:01:49,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 350 current snap is 0 2014-07-01 17:01:49,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: seq_num 1915 ranges {2297856=794624,3092480=659456,3751936=552960,4304896=393216} 2014-07-01 17:01:49,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: writing mira0435601-350 from 2297856 to 3092480 tid 1 2014-07-01 17:01:49,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: writing mira0435601-350 from 3092480 to 3751936 tid 2 2014-07-01 17:01:49,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: writing mira0435601-350 from 3751936 to 4304896 tid 3 2014-07-01 17:01:49,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: writing mira0435601-350 from 4304896 to 4698112 tid 4 2014-07-01 17:01:49,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: read oid 149 snap -1 2014-07-01 17:01:49,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: finishing write tid 1 to mira0435601-421 2014-07-01 17:01:49,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: finishing write tid 2 to mira0435601-421 2014-07-01 17:01:49,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: finishing write tid 3 to mira0435601-421 2014-07-01 17:01:49,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: finishing write tid 5 to mira0435601-421 2014-07-01 17:01:49,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: finishing write tid 6 to mira0435601-421 2014-07-01 17:01:49,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 294 (ObjNum 1913 snap 0 seq_num 1913) dirty exists 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: finishing write tid 1 to mira0435601-348 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: finishing write tid 2 to mira0435601-348 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: finishing write tid 4 to mira0435601-348 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: done (3 left) 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3072: read oid 275 snap -1 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: finishing write tid 5 to mira0435601-348 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 291 (ObjNum 1914 snap 0 seq_num 1914) dirty exists 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 1 to mira0435601-350 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 2 to mira0435601-350 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: done (3 left) 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 195 current snap is 0 2014-07-01 17:01:49,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: seq_num 1916 ranges {2818048=712704} 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: writing mira0435601-195 from 2818048 to 3530752 tid 1 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: read oid 308 snap -1 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 459 current snap is 0 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: seq_num 1917 ranges {6868992=585728} 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: writing mira0435601-459 from 6868992 to 7454720 tid 1 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 178 current snap is 0 2014-07-01 17:01:49,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: seq_num 1918 ranges {233472=614400,847872=655360,1503232=548864} 2014-07-01 17:01:49,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: writing mira0435601-178 from 233472 to 847872 tid 1 2014-07-01 17:01:49,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: writing mira0435601-178 from 847872 to 1503232 tid 2 2014-07-01 17:01:49,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: writing mira0435601-178 from 1503232 to 2052096 tid 3 2014-07-01 17:01:49,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: read oid 382 snap -1 2014-07-01 17:01:49,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 3 to mira0435601-350 2014-07-01 17:01:49,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 4 to mira0435601-350 2014-07-01 17:01:49,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 6 to mira0435601-350 2014-07-01 17:01:49,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: finishing write tid 7 to mira0435601-350 2014-07-01 17:01:49,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 283 (ObjNum 1915 snap 0 seq_num 1915) dirty exists 2014-07-01 17:01:49,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: expect (ObjNum 1239 snap 0 seq_num 1239) 2014-07-01 17:01:49,626.626 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:49,626.626 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:49,626.626 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:01:49,626.626 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:01:49,626.626 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:01:49,627.627 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:01:50,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 255 v 1017 (ObjNum 254 snap 0 seq_num 254) dirty exists 2014-07-01 17:01:50,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: finishing write tid 1 to mira0437250-62 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: finishing write tid 3 to mira0437250-62 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1755: done (10 left) 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1756: done (9 left) 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1764: done (8 left) 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1766: done (7 left) 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: rollback oid 332 current snap is 175 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 332 to 168 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 1 to mira0437250-383 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 3 to mira0437250-383 2014-07-01 17:01:50,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: finishing write tid 4 to mira0437250-62 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 1020 (ObjNum 958 snap 175 seq_num 958) dirty exists 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: expect (ObjNum 582 snap 31 seq_num 582) 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 1 to mira0437250-476 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 3 to mira0437250-476 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 660 (ObjNum 359 snap 0 seq_num 359) dirty exists 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 378 v 809 (ObjNum 686 snap 83 seq_num 686) dirty exists 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: finishing write tid 4 to mira0437250-476 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 1057 (ObjNum 957 snap 175 seq_num 957) dirty exists 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: finishing write tid 4 to mira0437250-383 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 755 (ObjNum 959 snap 175 seq_num 959) dirty exists 2014-07-01 17:01:50,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: expect (ObjNum 931 snap 167 seq_num 931) 2014-07-01 17:01:50,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 332 v 0 (ObjNum 18854712 snap 0 seq_num 35043728) dirty dne 2014-07-01 17:01:50,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1757: done (7 left) 2014-07-01 17:01:50,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1759: done (6 left) 2014-07-01 17:01:50,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1760: done (5 left) 2014-07-01 17:01:50,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:1761: done (4 left) 2014-07-01 17:01:50,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:1762: done (3 left) 2014-07-01 17:01:50,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:1763: done (2 left) 2014-07-01 17:01:50,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:1765: done (1 left) 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1767: done (0 left) 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: read oid 92 snap -1 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: read oid 22 snap -1 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: rollback oid 78 current snap is 175 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 78 to 156 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: write oid 404 current snap is 175 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: seq_num 960 ranges {737415=436201,1537414=1} 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: writing mira0437250-404 from 737415 to 1173616 tid 1 2014-07-01 17:01:51,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: writing mira0437250-404 from 1537414 to 1537415 tid 2 2014-07-01 17:01:51,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: read oid 236 snap 169 2014-07-01 17:01:51,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 0 (ObjNum 1543510256 snap 32691 seq_num 1543507232) dirty dne 2014-07-01 17:01:51,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: expect (ObjNum 91 snap 0 seq_num 91) 2014-07-01 17:01:51,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: expect (ObjNum 21 snap 0 seq_num 21) 2014-07-01 17:01:51,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: finishing write tid 1 to mira0437250-404 2014-07-01 17:01:51,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: finishing write tid 2 to mira0437250-404 2014-07-01 17:01:51,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: finishing write tid 4 to mira0437250-404 2014-07-01 17:01:51,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: finishing write tid 5 to mira0437250-404 2014-07-01 17:01:51,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 497 (ObjNum 960 snap 175 seq_num 960) dirty exists 2014-07-01 17:01:51,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: expect (ObjNum 235 snap 0 seq_num 235) 2014-07-01 17:01:51,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: finishing write tid 1 to mira0435601-195 2014-07-01 17:01:51,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: finishing write tid 3 to mira0435601-195 2014-07-01 17:01:51,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: expect (ObjNum 1314 snap 0 seq_num 1314) 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: finishing write tid 4 to mira0435601-195 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 266 (ObjNum 1916 snap 0 seq_num 1916) dirty exists 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 1 to mira0435601-459 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 3 to mira0435601-459 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 4 to mira0435601-459 2014-07-01 17:01:53,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 338 (ObjNum 1917 snap 0 seq_num 1917) dirty exists 2014-07-01 17:01:53,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: finishing write tid 1 to mira0435601-178 2014-07-01 17:01:53,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: finishing write tid 2 to mira0435601-178 2014-07-01 17:01:53,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: finishing write tid 3 to mira0435601-178 2014-07-01 17:01:53,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: finishing write tid 5 to mira0435601-178 2014-07-01 17:01:53,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: expect (ObjNum 1391 snap 0 seq_num 1391) 2014-07-01 17:01:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1768: done (4 left) 2014-07-01 17:01:53,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1769: done (3 left) 2014-07-01 17:01:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1770: done (2 left) 2014-07-01 17:01:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1771: done (1 left) 2014-07-01 17:01:53,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:1772: done (0 left) 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1773: rollback oid 79 current snap is 175 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 79 to 140 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: snap_create 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 1069 (ObjNum 715 snap 96 seq_num 715) dirty exists 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: finishing write tid 6 to mira0435601-178 2014-07-01 17:01:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 342 (ObjNum 1918 snap 0 seq_num 1918) dirty exists 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: done (7 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: done (6 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3072: done (5 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: done (4 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: done (3 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: done (2 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: done (1 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: done (0 left) 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 473 current snap is 0 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: seq_num 1919 ranges {2371584=483328,2854912=483328,3338240=602112,3940352=118784} 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: writing mira0435601-473 from 2371584 to 2854912 tid 1 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: writing mira0435601-473 from 2854912 to 3338240 tid 2 2014-07-01 17:01:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: writing mira0435601-473 from 3338240 to 3940352 tid 3 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: writing mira0435601-473 from 3940352 to 4059136 tid 4 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 454 current snap is 0 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: seq_num 1920 ranges {3207168=757760,3964928=667648,4632576=135168} 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: writing mira0435601-454 from 3207168 to 3964928 tid 1 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: writing mira0435601-454 from 3964928 to 4632576 tid 2 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: writing mira0435601-454 from 4632576 to 4767744 tid 3 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3080: read oid 399 snap -1 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 1 to mira0435601-473 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 2 to mira0435601-473 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 3 to mira0435601-473 2014-07-01 17:01:53,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 4 to mira0435601-473 2014-07-01 17:01:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 6 to mira0435601-473 2014-07-01 17:01:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: finishing write tid 1 to mira0435601-454 2014-07-01 17:01:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 278 current snap is 0 2014-07-01 17:01:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: seq_num 1921 ranges {5521408=434176,5955584=651264,6606848=151552} 2014-07-01 17:01:53,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: writing mira0435601-278 from 5521408 to 5955584 tid 1 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: writing mira0435601-278 from 5955584 to 6606848 tid 2 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: writing mira0435601-278 from 6606848 to 6758400 tid 3 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 240 current snap is 0 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: seq_num 1922 ranges {3108864=57344} 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: writing mira0435601-240 from 3108864 to 3166208 tid 1 2014-07-01 17:01:53,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3083: read oid 482 snap -1 2014-07-01 17:01:53,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: finishing write tid 2 to mira0435601-454 2014-07-01 17:01:53,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: finishing write tid 3 to mira0435601-454 2014-07-01 17:01:53,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: finishing write tid 5 to mira0435601-454 2014-07-01 17:01:53,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3080: expect (ObjNum 1753 snap 0 seq_num 1753) 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: finishing write tid 6 to mira0435601-454 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 183 (ObjNum 1920 snap 0 seq_num 1920) dirty exists 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: finishing write tid 1 to mira0435601-278 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: finishing write tid 2 to mira0435601-278 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: finishing write tid 3 to mira0435601-278 2014-07-01 17:01:53,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: finishing write tid 5 to mira0435601-278 2014-07-01 17:01:53,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: done (5 left) 2014-07-01 17:01:53,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3080: done (4 left) 2014-07-01 17:01:53,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 439 current snap is 0 2014-07-01 17:01:53,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: seq_num 1923 ranges {13041664=577536,13619200=401408} 2014-07-01 17:01:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: writing mira0435601-439 from 13041664 to 13619200 tid 1 2014-07-01 17:01:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: writing mira0435601-439 from 13619200 to 14020608 tid 2 2014-07-01 17:01:54,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 254 current snap is 0 2014-07-01 17:01:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: seq_num 1924 ranges {3645440=606208,4251648=647168,4898816=720896,5619712=237568} 2014-07-01 17:01:54,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: writing mira0435601-254 from 3645440 to 4251648 tid 1 2014-07-01 17:01:54,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: writing mira0435601-254 from 4251648 to 4898816 tid 2 2014-07-01 17:01:54,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: writing mira0435601-254 from 4898816 to 5619712 tid 3 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: writing mira0435601-254 from 5619712 to 5857280 tid 4 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: delete oid 204 current snap is 0 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: finishing write tid 6 to mira0435601-278 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 278 v 300 (ObjNum 1921 snap 0 seq_num 1921) dirty exists 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: finishing write tid 1 to mira0435601-240 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: finishing write tid 3 to mira0435601-240 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: finishing write tid 4 to mira0435601-240 2014-07-01 17:01:54,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 160 (ObjNum 1922 snap 0 seq_num 1922) dirty exists 2014-07-01 17:01:54,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:3083: expect (ObjNum 1100 snap 0 seq_num 1100) 2014-07-01 17:01:54,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1773: done (1 left) 2014-07-01 17:01:54,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1774: done (0 left) 2014-07-01 17:01:54,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: snap_create 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 1 to mira0435601-439 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 2 to mira0435601-439 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 4 to mira0435601-439 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 1 to mira0435601-254 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 2 to mira0435601-254 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 3 to mira0435601-254 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 4 to mira0435601-254 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 6 to mira0435601-254 2014-07-01 17:01:54,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: finishing write tid 7 to mira0435601-254 2014-07-01 17:01:54,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 221 (ObjNum 1924 snap 0 seq_num 1924) dirty exists 2014-07-01 17:01:54,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: done (6 left) 2014-07-01 17:01:54,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: done (5 left) 2014-07-01 17:01:54,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3083: done (4 left) 2014-07-01 17:01:54,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: done (3 left) 2014-07-01 17:01:54,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: done (2 left) 2014-07-01 17:01:54,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3087: read oid 309 snap -1 2014-07-01 17:01:54,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3088: read oid 280 snap -1 2014-07-01 17:01:54,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3089: read oid 323 snap -1 2014-07-01 17:01:54,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: read oid 60 snap -1 2014-07-01 17:01:54,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3091: read oid 140 snap -1 2014-07-01 17:01:54,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3092: read oid 242 snap -1 2014-07-01 17:01:54,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 347 current snap is 0 2014-07-01 17:01:54,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: seq_num 1926 ranges {5722112=397312} 2014-07-01 17:01:54,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: writing mira0435601-347 from 5722112 to 6119424 tid 1 2014-07-01 17:01:54,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: read oid 132 snap -1 2014-07-01 17:01:54,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3092: expect (ObjNum 742 snap 0 seq_num 742) 2014-07-01 17:01:55,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1775: done (0 left) 2014-07-01 17:01:55,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1776: rollback oid 358 current snap is 177 2014-07-01 17:01:55,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 358 to 175 2014-07-01 17:01:55,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1777: rollback oid 172 current snap is 177 2014-07-01 17:01:55,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 172 to 149 2014-07-01 17:01:55,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1778: snap_create 2014-07-01 17:01:55,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 701 (ObjNum 357 snap 0 seq_num 357) dirty exists 2014-07-01 17:01:55,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 780 (ObjNum 171 snap 0 seq_num 171) dirty exists 2014-07-01 17:01:55,605.605 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:01:55,606.606 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:01:55,606.606 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_2 2014-07-01 17:01:55,606.606 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:01:55,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: expect (ObjNum 1199 snap 0 seq_num 1199) 2014-07-01 17:01:56,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:1776: done (2 left) 2014-07-01 17:01:56,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1777: done (1 left) 2014-07-01 17:01:56,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1778: done (0 left) 2014-07-01 17:01:56,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: delete oid 426 current snap is 178 2014-07-01 17:01:56,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1779: done (0 left) 2014-07-01 17:01:56,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1780: delete oid 210 current snap is 178 2014-07-01 17:01:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1780: done (0 left) 2014-07-01 17:01:56,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: delete oid 485 current snap is 178 2014-07-01 17:01:56,392.392 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:01:56,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1781: done (0 left) 2014-07-01 17:01:56,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: write oid 159 current snap is 178 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: seq_num 964 ranges {532862=511962,1635411=626670,2932861=1} 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0437250-159 from 532862 to 1044824 tid 1 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0437250-159 from 1635411 to 2262081 tid 2 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: writing mira0437250-159 from 2932861 to 2932862 tid 3 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: write oid 471 current snap is 178 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: seq_num 965 ranges {194043=1} 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: writing mira0437250-471 from 194043 to 194044 tid 1 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: delete oid 466 current snap is 178 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 1 to mira0437250-159 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1784: done (2 left) 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: read oid 147 snap 168 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1786: delete oid 325 current snap is 178 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: finishing write tid 1 to mira0437250-471 2014-07-01 17:01:56,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: finishing write tid 3 to mira0437250-471 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 2 to mira0437250-159 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 3 to mira0437250-159 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 5 to mira0437250-159 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: finishing write tid 6 to mira0437250-159 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 159 v 768 (ObjNum 964 snap 178 seq_num 964) dirty exists 2014-07-01 17:01:56,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: expect (ObjNum 884 snap 150 seq_num 884) 2014-07-01 17:01:56,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: finishing write tid 4 to mira0437250-471 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 667 (ObjNum 965 snap 178 seq_num 965) dirty exists 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1782: done (3 left) 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1783: done (2 left) 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1785: done (1 left) 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1786: done (0 left) 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: write oid 282 current snap is 178 2014-07-01 17:01:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: seq_num 968 ranges {577909=435275,1377908=1} 2014-07-01 17:01:56,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: writing mira0437250-282 from 577909 to 1013184 tid 1 2014-07-01 17:01:56,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: writing mira0437250-282 from 1377908 to 1377909 tid 2 2014-07-01 17:01:56,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: write oid 80 current snap is 178 2014-07-01 17:01:56,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: seq_num 969 ranges {522739=408104,1680632=442107} 2014-07-01 17:01:56,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: writing mira0437250-80 from 522739 to 930843 tid 1 2014-07-01 17:01:56,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: writing mira0437250-80 from 1680632 to 2122739 tid 2 2014-07-01 17:01:56,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1789: read oid 415 snap 156 2014-07-01 17:01:56,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 1 to mira0437250-282 2014-07-01 17:01:56,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 2 to mira0437250-282 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 4 to mira0437250-282 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: finishing write tid 5 to mira0437250-282 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 750 (ObjNum 968 snap 178 seq_num 968) dirty exists 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 1 to mira0437250-80 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1787: done (2 left) 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1790: snap_create 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 2 to mira0437250-80 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 4 to mira0437250-80 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: finishing write tid 5 to mira0437250-80 2014-07-01 17:01:56,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 864 (ObjNum 969 snap 178 seq_num 969) dirty exists 2014-07-01 17:01:57,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3087: expect (ObjNum 1780 snap 0 seq_num 1780) 2014-07-01 17:01:57,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1788: done (2 left) 2014-07-01 17:01:57,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1789: done (1 left) 2014-07-01 17:01:57,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1790: done (0 left) 2014-07-01 17:01:57,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: read oid 137 snap -1 2014-07-01 17:01:57,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: snap_remove snap 163 2014-07-01 17:01:57,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: expect (ObjNum 651 snap 71 seq_num 651) 2014-07-01 17:01:58,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:1791: done (1 left) 2014-07-01 17:01:58,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:1792: done (0 left) 2014-07-01 17:01:58,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: write oid 492 current snap is 179 2014-07-01 17:01:58,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: seq_num 970 ranges {583921=531332,1911116=550689,3119127=264794} 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0437250-492 from 583921 to 1115253 tid 1 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0437250-492 from 1911116 to 2461805 tid 2 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: writing mira0437250-492 from 3119127 to 3383921 tid 3 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: read oid 43 snap -1 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: write oid 482 current snap is 179 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: seq_num 971 ranges {679150=503136,1589151=620884,2733855=345295} 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: writing mira0437250-482 from 679150 to 1182286 tid 1 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: writing mira0437250-482 from 1589151 to 2210035 tid 2 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: writing mira0437250-482 from 2733855 to 3079150 tid 3 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: snap_create 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: expect (ObjNum 500 snap 1 seq_num 500) 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 1 to mira0437250-492 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 2 to mira0437250-492 2014-07-01 17:01:58,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 3 to mira0437250-492 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 5 to mira0437250-492 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: finishing write tid 6 to mira0437250-492 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 737 (ObjNum 970 snap 179 seq_num 970) dirty exists 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: finishing write tid 1 to mira0437250-482 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: finishing write tid 2 to mira0437250-482 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: finishing write tid 3 to mira0437250-482 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: finishing write tid 5 to mira0437250-482 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: finishing write tid 6 to mira0437250-482 2014-07-01 17:01:58,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 939 (ObjNum 971 snap 179 seq_num 971) dirty exists 2014-07-01 17:01:58,536.536 INFO:teuthology.task.rados.rados.0.mira043.stdout:3091: expect (ObjNum 1804 snap 0 seq_num 1804) 2014-07-01 17:01:59,278.278 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:01:59,279.279 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pg_num 46' 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1793: done (3 left) 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1794: done (2 left) 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1795: done (1 left) 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1796: done (0 left) 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: write oid 114 current snap is 180 2014-07-01 17:01:59,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: seq_num 972 ranges {774380=510013,1737332=690055,2774379=1} 2014-07-01 17:01:59,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: writing mira0437250-114 from 774380 to 1284393 tid 1 2014-07-01 17:01:59,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: writing mira0437250-114 from 1737332 to 2427387 tid 2 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: writing mira0437250-114 from 2774379 to 2774380 tid 3 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: write oid 184 current snap is 180 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: seq_num 973 ranges {566761=1} 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: writing mira0437250-184 from 566761 to 566762 tid 1 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: snap_remove snap 167 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: finishing write tid 1 to mira0437250-114 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 1 to mira0437250-184 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 3 to mira0437250-184 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: finishing write tid 4 to mira0437250-184 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 557 (ObjNum 973 snap 180 seq_num 973) dirty exists 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: finishing write tid 2 to mira0437250-114 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: finishing write tid 3 to mira0437250-114 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: finishing write tid 5 to mira0437250-114 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: finishing write tid 6 to mira0437250-114 2014-07-01 17:01:59,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 1072 (ObjNum 972 snap 180 seq_num 972) dirty exists 2014-07-01 17:02:00,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:1797: done (2 left) 2014-07-01 17:02:00,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:1798: done (1 left) 2014-07-01 17:02:00,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:1799: done (0 left) 2014-07-01 17:02:00,385.385 INFO:teuthology.task.rados.rados.0.mira043.stdout:1800: snap_create 2014-07-01 17:02:00,393.393 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pg_num to 46 2014-07-01 17:02:01,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1800: done (0 left) 2014-07-01 17:02:01,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: write oid 55 current snap is 181 2014-07-01 17:02:01,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: seq_num 974 ranges {779367=1} 2014-07-01 17:02:01,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: writing mira0437250-55 from 779367 to 779368 tid 1 2014-07-01 17:02:01,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: rollback oid 371 current snap is 181 2014-07-01 17:02:01,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 371 to 168 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: read oid 439 snap 173 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: rollback oid 227 current snap is 181 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 227 to 173 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1805: delete oid 299 current snap is 181 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1805: done (4 left) 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1806: read oid 73 snap -1 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: write oid 131 current snap is 181 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: seq_num 976 ranges {668003=584750,1970245=297758} 2014-07-01 17:02:01,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: writing mira0437250-131 from 668003 to 1252753 tid 1 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: writing mira0437250-131 from 1970245 to 2268003 tid 2 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1808: snap_remove snap 176 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 1 to mira0437250-55 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 1141 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 3 to mira0437250-55 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 0 (ObjNum 15636784 snap 0 seq_num 15649600) dirty dne 2014-07-01 17:02:01,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: finishing write tid 1 to mira0437250-131 2014-07-01 17:02:01,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: finishing write tid 2 to mira0437250-131 2014-07-01 17:02:01,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: finishing write tid 4 to mira0437250-131 2014-07-01 17:02:01,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: finishing write tid 5 to mira0437250-131 2014-07-01 17:02:01,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 676 (ObjNum 976 snap 181 seq_num 976) dirty exists 2014-07-01 17:02:01,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:1806: expect (ObjNum 72 snap 0 seq_num 72) 2014-07-01 17:02:02,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: finishing write tid 4 to mira0437250-55 2014-07-01 17:02:02,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 1330 (ObjNum 974 snap 181 seq_num 974) dirty exists 2014-07-01 17:02:02,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: expect (ObjNum 438 snap 0 seq_num 438) 2014-07-01 17:02:03,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1801: done (6 left) 2014-07-01 17:02:03,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1802: done (5 left) 2014-07-01 17:02:03,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1803: done (4 left) 2014-07-01 17:02:03,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1804: done (3 left) 2014-07-01 17:02:03,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:1806: done (2 left) 2014-07-01 17:02:03,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1807: done (1 left) 2014-07-01 17:02:03,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1808: done (0 left) 2014-07-01 17:02:03,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: write oid 124 current snap is 181 2014-07-01 17:02:03,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: seq_num 977 ranges {764257=557578,1933977=30280} 2014-07-01 17:02:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: writing mira0437250-124 from 764257 to 1321835 tid 1 2014-07-01 17:02:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: writing mira0437250-124 from 1933977 to 1964257 tid 2 2014-07-01 17:02:03,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: read oid 132 snap -1 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: delete oid 265 current snap is 181 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 1 to mira0437250-124 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 2 to mira0437250-124 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 4 to mira0437250-124 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1810: done (2 left) 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1811: done (1 left) 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: read oid 303 snap -1 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: write oid 215 current snap is 181 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: seq_num 979 ranges {769244=501187,1691073=719342,2972012=645166,3969243=1} 2014-07-01 17:02:03,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: writing mira0437250-215 from 769244 to 1270431 tid 1 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: writing mira0437250-215 from 1691073 to 2410415 tid 2 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: writing mira0437250-215 from 2972012 to 3617178 tid 3 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: writing mira0437250-215 from 3969243 to 3969244 tid 4 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1814: snap_create 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: finishing write tid 5 to mira0437250-124 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 1046 (ObjNum 977 snap 181 seq_num 977) dirty exists 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 1 to mira0437250-215 2014-07-01 17:02:03,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: expect (ObjNum 302 snap 0 seq_num 302) 2014-07-01 17:02:03,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3088: expect (ObjNum 1752 snap 0 seq_num 1752) 2014-07-01 17:02:05,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 2 to mira0437250-215 2014-07-01 17:02:05,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 3 to mira0437250-215 2014-07-01 17:02:05,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 4 to mira0437250-215 2014-07-01 17:02:05,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 6 to mira0437250-215 2014-07-01 17:02:05,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: finishing write tid 7 to mira0437250-215 2014-07-01 17:02:05,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 1030 (ObjNum 979 snap 181 seq_num 979) dirty exists 2014-07-01 17:02:05,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1809: done (3 left) 2014-07-01 17:02:05,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1812: done (2 left) 2014-07-01 17:02:05,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1813: done (1 left) 2014-07-01 17:02:05,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:1814: done (0 left) 2014-07-01 17:02:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1815: read oid 195 snap -1 2014-07-01 17:02:05,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: snap_create 2014-07-01 17:02:05,405.405 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:05,405.405 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:05,405.405 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_1 2014-07-01 17:02:05,405.405 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:02:05,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1815: expect (ObjNum 757 snap 111 seq_num 757) 2014-07-01 17:02:06,167.167 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:02:06,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:1815: done (1 left) 2014-07-01 17:02:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1816: done (0 left) 2014-07-01 17:02:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: write oid 174 current snap is 183 2014-07-01 17:02:06,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: seq_num 980 ranges {464474=509087,1687829=789537,2904437=760037} 2014-07-01 17:02:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: writing mira0437250-174 from 464474 to 973561 tid 1 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: writing mira0437250-174 from 1687829 to 2477366 tid 2 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: writing mira0437250-174 from 2904437 to 3664474 tid 3 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1818: delete oid 188 current snap is 183 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: finishing write tid 1 to mira0437250-174 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: finishing write tid 2 to mira0437250-174 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1818: done (1 left) 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1819: delete oid 149 current snap is 183 2014-07-01 17:02:06,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:1819: done (1 left) 2014-07-01 17:02:06,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1820: snap_create 2014-07-01 17:02:06,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: finishing write tid 3 to mira0437250-174 2014-07-01 17:02:06,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: finishing write tid 5 to mira0437250-174 2014-07-01 17:02:06,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: finishing write tid 6 to mira0437250-174 2014-07-01 17:02:06,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 174 v 750 (ObjNum 980 snap 183 seq_num 980) dirty exists 2014-07-01 17:02:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1817: done (1 left) 2014-07-01 17:02:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1820: done (0 left) 2014-07-01 17:02:07,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:1821: snap_create 2014-07-01 17:02:07,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: finishing write tid 1 to mira0435601-347 2014-07-01 17:02:07,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 17:02:07,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: finishing write tid 3 to mira0435601-347 2014-07-01 17:02:07,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: finishing write tid 7 to mira0435601-473 2014-07-01 17:02:07,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 252 (ObjNum 1919 snap 0 seq_num 1919) dirty exists 2014-07-01 17:02:07,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 5 to mira0435601-439 2014-07-01 17:02:07,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 439 v 297 (ObjNum 1923 snap 0 seq_num 1923) dirty exists 2014-07-01 17:02:07,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: finishing write tid 4 to mira0435601-347 2014-07-01 17:02:07,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 268 (ObjNum 1926 snap 0 seq_num 1926) dirty exists 2014-07-01 17:02:07,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: done (9 left) 2014-07-01 17:02:07,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: done (8 left) 2014-07-01 17:02:07,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3087: done (7 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3088: done (6 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3089: done (5 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: done (4 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3091: done (3 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3092: done (2 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: done (1 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: done (0 left) 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 17:02:07,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: seq_num 1927 ranges {6389760=425984,6815744=446464,7262208=495616,7757824=262144} 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: writing mira0435601-176 from 6389760 to 6815744 tid 1 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: writing mira0435601-176 from 6815744 to 7262208 tid 2 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: writing mira0435601-176 from 7262208 to 7757824 tid 3 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: writing mira0435601-176 from 7757824 to 8019968 tid 4 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 28 current snap is 0 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: seq_num 1928 ranges {0=450560} 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: writing mira0435601-28 from 0 to 450560 tid 1 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: seq_num 1929 ranges {2113536=569344,2682880=454656,3137536=348160} 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0435601-418 from 2113536 to 2682880 tid 1 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0435601-418 from 2682880 to 3137536 tid 2 2014-07-01 17:02:07,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0435601-418 from 3137536 to 3485696 tid 3 2014-07-01 17:02:07,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 210 current snap is 0 2014-07-01 17:02:07,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: seq_num 1930 ranges {3010560=647168,3657728=401408} 2014-07-01 17:02:07,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: writing mira0435601-210 from 3010560 to 3657728 tid 1 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: writing mira0435601-210 from 3657728 to 4059136 tid 2 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: delete oid 254 current snap is 0 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 1 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 2 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 3 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 4 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 6 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: finishing write tid 7 to mira0435601-176 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 256 (ObjNum 1927 snap 0 seq_num 1927) dirty exists 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: finishing write tid 1 to mira0435601-28 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: finishing write tid 3 to mira0435601-28 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: finishing write tid 4 to mira0435601-28 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 28 v 162 (ObjNum 1928 snap 0 seq_num 1928) dirty exists 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 1 to mira0435601-418 2014-07-01 17:02:08,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 2 to mira0435601-418 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 3 to mira0435601-418 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 5 to mira0435601-418 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 6 to mira0435601-418 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 317 (ObjNum 1929 snap 0 seq_num 1929) dirty exists 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: finishing write tid 1 to mira0435601-210 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: finishing write tid 2 to mira0435601-210 2014-07-01 17:02:08,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: finishing write tid 4 to mira0435601-210 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: done (4 left) 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: done (3 left) 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: done (2 left) 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: done (1 left) 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3100: read oid 32 snap -1 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 84 current snap is 0 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: seq_num 1932 ranges {2146304=790528} 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: writing mira0435601-84 from 2146304 to 2936832 tid 1 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 367 current snap is 0 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: seq_num 1933 ranges {3280896=417792,3698688=438272,4136960=733184,4870144=434176} 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: writing mira0435601-367 from 3280896 to 3698688 tid 1 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: writing mira0435601-367 from 3698688 to 4136960 tid 2 2014-07-01 17:02:08,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: writing mira0435601-367 from 4136960 to 4870144 tid 3 2014-07-01 17:02:08,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: writing mira0435601-367 from 4870144 to 5304320 tid 4 2014-07-01 17:02:08,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 408 current snap is 0 2014-07-01 17:02:08,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: seq_num 1934 ranges {4239360=491520,4730880=667648,5398528=536576} 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: writing mira0435601-408 from 4239360 to 4730880 tid 1 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: writing mira0435601-408 from 4730880 to 5398528 tid 2 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: writing mira0435601-408 from 5398528 to 5935104 tid 3 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 475 current snap is 0 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: seq_num 1935 ranges {3129344=765952,3895296=651264,4546560=151552} 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: writing mira0435601-475 from 3129344 to 3895296 tid 1 2014-07-01 17:02:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: writing mira0435601-475 from 3895296 to 4546560 tid 2 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: writing mira0435601-475 from 4546560 to 4698112 tid 3 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3105: read oid 337 snap -1 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: finishing write tid 5 to mira0435601-210 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 210 v 364 (ObjNum 1930 snap 0 seq_num 1930) dirty exists 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: finishing write tid 1 to mira0435601-84 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: finishing write tid 3 to mira0435601-84 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: finishing write tid 4 to mira0435601-84 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 299 (ObjNum 1932 snap 0 seq_num 1932) dirty exists 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 1 to mira0435601-367 2014-07-01 17:02:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: done (6 left) 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3100: done (5 left) 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: done (4 left) 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: read oid 343 snap -1 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 2 to mira0435601-367 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 3 to mira0435601-367 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 4 to mira0435601-367 2014-07-01 17:02:08,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 6 to mira0435601-367 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: finishing write tid 7 to mira0435601-367 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 267 (ObjNum 1933 snap 0 seq_num 1933) dirty exists 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: finishing write tid 1 to mira0435601-408 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: finishing write tid 2 to mira0435601-408 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: finishing write tid 3 to mira0435601-408 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: finishing write tid 5 to mira0435601-408 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: finishing write tid 6 to mira0435601-408 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 303 (ObjNum 1934 snap 0 seq_num 1934) dirty exists 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: done (4 left) 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: done (3 left) 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3107: read oid 224 snap -1 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3108: read oid 56 snap -1 2014-07-01 17:02:08,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: finishing write tid 1 to mira0435601-475 2014-07-01 17:02:08,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 328 current snap is 0 2014-07-01 17:02:08,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: seq_num 1936 ranges {6008832=634880,6643712=434176,7077888=368640} 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: writing mira0435601-328 from 6008832 to 6643712 tid 1 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: writing mira0435601-328 from 6643712 to 7077888 tid 2 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: writing mira0435601-328 from 7077888 to 7446528 tid 3 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: seq_num 1937 ranges {3416064=262144} 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: writing mira0435601-445 from 3416064 to 3678208 tid 1 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 62 current snap is 0 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: seq_num 1938 ranges {1814528=135168} 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: writing mira0435601-62 from 1814528 to 1949696 tid 1 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: read oid 132 snap -1 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: finishing write tid 2 to mira0435601-475 2014-07-01 17:02:08,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: expect (ObjNum 1495 snap 0 seq_num 1495) 2014-07-01 17:02:08,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1821: done (0 left) 2014-07-01 17:02:08,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: read oid 118 snap -1 2014-07-01 17:02:08,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: rollback oid 228 current snap is 185 2014-07-01 17:02:08,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 228 to 168 2014-07-01 17:02:08,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:1824: read oid 36 snap -1 2014-07-01 17:02:08,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:1825: snap_remove snap 169 2014-07-01 17:02:08,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 1076 (ObjNum 227 snap 0 seq_num 227) dirty exists 2014-07-01 17:02:08,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: expect (ObjNum 739 snap 103 seq_num 739) 2014-07-01 17:02:08,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: finishing write tid 3 to mira0435601-475 2014-07-01 17:02:08,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:3108: expect (ObjNum 1568 snap 0 seq_num 1568) 2014-07-01 17:02:09,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1822: done (3 left) 2014-07-01 17:02:09,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1823: done (2 left) 2014-07-01 17:02:09,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1824: done (1 left) 2014-07-01 17:02:09,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:1825: done (0 left) 2014-07-01 17:02:09,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: write oid 421 current snap is 185 2014-07-01 17:02:09,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: seq_num 983 ranges {564691=575924,1364690=1} 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: writing mira0437250-421 from 564691 to 1140615 tid 1 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: writing mira0437250-421 from 1364690 to 1364691 tid 2 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1827: snap_remove snap 155 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 1 to mira0437250-421 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 2 to mira0437250-421 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 4 to mira0437250-421 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: finishing write tid 5 to mira0437250-421 2014-07-01 17:02:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 1337 (ObjNum 983 snap 185 seq_num 983) dirty exists 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: finishing write tid 5 to mira0435601-475 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: finishing write tid 6 to mira0435601-475 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 287 (ObjNum 1935 snap 0 seq_num 1935) dirty exists 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: finishing write tid 1 to mira0435601-62 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: finishing write tid 3 to mira0435601-62 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: done (8 left) 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: done (7 left) 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3108: done (6 left) 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: read oid 19 snap -1 2014-07-01 17:02:10,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: finishing write tid 4 to mira0435601-62 2014-07-01 17:02:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 231 (ObjNum 1938 snap 0 seq_num 1938) dirty exists 2014-07-01 17:02:10,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3107: expect (ObjNum 1398 snap 0 seq_num 1398) 2014-07-01 17:02:10,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1826: done (1 left) 2014-07-01 17:02:10,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1827: done (0 left) 2014-07-01 17:02:10,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: snap_create 2014-07-01 17:02:11,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1828: done (0 left) 2014-07-01 17:02:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: read oid 234 snap 175 2014-07-01 17:02:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: read oid 356 snap 181 2014-07-01 17:02:11,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:1831: read oid 266 snap -1 2014-07-01 17:02:11,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1832: snap_create 2014-07-01 17:02:11,659.659 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: expect (ObjNum 355 snap 0 seq_num 355) 2014-07-01 17:02:11,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: expect (ObjNum 233 snap 0 seq_num 233) 2014-07-01 17:02:12,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:1831: expect (ObjNum 265 snap 0 seq_num 265) 2014-07-01 17:02:12,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1829: done (3 left) 2014-07-01 17:02:12,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1830: done (2 left) 2014-07-01 17:02:12,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1831: done (1 left) 2014-07-01 17:02:12,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1832: done (0 left) 2014-07-01 17:02:12,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: delete oid 340 current snap is 187 2014-07-01 17:02:12,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1833: done (0 left) 2014-07-01 17:02:12,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: write oid 367 current snap is 187 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: seq_num 985 ranges {605775=555629,1787322=18453} 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: writing mira0437250-367 from 605775 to 1161404 tid 1 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: writing mira0437250-367 from 1787322 to 1805775 tid 2 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: write oid 11 current snap is 187 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: seq_num 986 ranges {549580=678857,1865357=597204,2549579=1} 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0437250-11 from 549580 to 1228437 tid 1 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0437250-11 from 1865357 to 2462561 tid 2 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: writing mira0437250-11 from 2549579 to 2549580 tid 3 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: write oid 210 current snap is 187 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: seq_num 987 ranges {645834=651686,1980514=265320} 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: writing mira0437250-210 from 645834 to 1297520 tid 1 2014-07-01 17:02:12,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: writing mira0437250-210 from 1980514 to 2245834 tid 2 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: read oid 212 snap 156 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 1 to mira0437250-367 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 2 to mira0437250-367 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 4 to mira0437250-367 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: finishing write tid 5 to mira0437250-367 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 1093 (ObjNum 985 snap 187 seq_num 985) dirty exists 2014-07-01 17:02:12,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 1 to mira0437250-11 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 2 to mira0437250-11 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 3 to mira0437250-11 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 5 to mira0437250-11 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: finishing write tid 6 to mira0437250-11 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 763 (ObjNum 986 snap 187 seq_num 986) dirty exists 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1834: done (3 left) 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1835: done (2 left) 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: write oid 327 current snap is 187 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: seq_num 988 ranges {705992=658562,1808950=489018,2762815=710568,3505991=1} 2014-07-01 17:02:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0437250-327 from 705992 to 1364554 tid 1 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0437250-327 from 1808950 to 2297968 tid 2 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0437250-327 from 2762815 to 3473383 tid 3 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: writing mira0437250-327 from 3505991 to 3505992 tid 4 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: write oid 280 current snap is 187 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: seq_num 989 ranges {249797=1} 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: writing mira0437250-280 from 249797 to 249798 tid 1 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: snap_create 2014-07-01 17:02:13,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 1 to mira0437250-210 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 2 to mira0437250-210 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 4 to mira0437250-210 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: finishing write tid 5 to mira0437250-210 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 210 v 686 (ObjNum 987 snap 187 seq_num 987) dirty exists 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 1 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 2 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 1 to mira0437250-280 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 3 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 3 to mira0437250-280 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 4 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 6 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: finishing write tid 4 to mira0437250-280 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 280 v 665 (ObjNum 989 snap 187 seq_num 989) dirty exists 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: finishing write tid 7 to mira0437250-327 2014-07-01 17:02:13,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 733 (ObjNum 988 snap 187 seq_num 988) dirty exists 2014-07-01 17:02:13,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1836: done (4 left) 2014-07-01 17:02:13,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1837: done (3 left) 2014-07-01 17:02:13,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:1838: done (2 left) 2014-07-01 17:02:13,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1839: done (1 left) 2014-07-01 17:02:13,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1840: done (0 left) 2014-07-01 17:02:13,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: write oid 394 current snap is 188 2014-07-01 17:02:13,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: seq_num 990 ranges {746051=602171,2133342=629409,3262418=683633} 2014-07-01 17:02:13,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: writing mira0437250-394 from 746051 to 1348222 tid 1 2014-07-01 17:02:13,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: writing mira0437250-394 from 2133342 to 2762751 tid 2 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: writing mira0437250-394 from 3262418 to 3946051 tid 3 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1842: rollback oid 116 current snap is 188 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 116 to 149 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: finishing write tid 1 to mira0437250-394 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: finishing write tid 2 to mira0437250-394 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: read oid 165 snap -1 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1844: rollback oid 33 current snap is 188 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 33 to 158 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: write oid 400 current snap is 188 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: seq_num 991 ranges {689856=1} 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: writing mira0437250-400 from 689856 to 689857 tid 1 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1846: snap_create 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 0 (ObjNum -1829423232 snap 32691 seq_num 1835018) dirty dne 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: finishing write tid 3 to mira0437250-394 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: finishing write tid 5 to mira0437250-394 2014-07-01 17:02:13,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 1 to mira0437250-400 2014-07-01 17:02:13,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 3 to mira0437250-400 2014-07-01 17:02:13,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 1155 (ObjNum 115 snap 0 seq_num 115) dirty exists 2014-07-01 17:02:13,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: finishing write tid 4 to mira0437250-400 2014-07-01 17:02:13,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 607 (ObjNum 991 snap 188 seq_num 991) dirty exists 2014-07-01 17:02:13,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: expect (ObjNum 164 snap 0 seq_num 164) 2014-07-01 17:02:13,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: finishing write tid 1 to mira0435601-328 2014-07-01 17:02:13,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: finishing write tid 2 to mira0435601-328 2014-07-01 17:02:13,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: finishing write tid 3 to mira0435601-328 2014-07-01 17:02:13,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: finishing write tid 5 to mira0435601-328 2014-07-01 17:02:13,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: finishing write tid 6 to mira0435601-328 2014-07-01 17:02:13,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 272 (ObjNum 1936 snap 0 seq_num 1936) dirty exists 2014-07-01 17:02:13,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: finishing write tid 1 to mira0435601-445 2014-07-01 17:02:13,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: finishing write tid 3 to mira0435601-445 2014-07-01 17:02:13,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: finishing write tid 4 to mira0435601-445 2014-07-01 17:02:13,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 234 (ObjNum 1937 snap 0 seq_num 1937) dirty exists 2014-07-01 17:02:13,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 17:02:13,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:3105: expect (ObjNum 1654 snap 0 seq_num 1654) 2014-07-01 17:02:14,201.201 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:14,201.201 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:14,202.202 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:02:14,202.202 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:02:14,747.747 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:02:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: finishing write tid 6 to mira0437250-394 2014-07-01 17:02:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 1069 (ObjNum 990 snap 188 seq_num 990) dirty exists 2014-07-01 17:02:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1841: done (5 left) 2014-07-01 17:02:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1842: done (4 left) 2014-07-01 17:02:15,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:1843: done (3 left) 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1844: done (2 left) 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1845: done (1 left) 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1846: done (0 left) 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: rollback oid 116 current snap is 189 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 116 to 188 2014-07-01 17:02:15,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: write oid 137 current snap is 189 2014-07-01 17:02:15,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: seq_num 992 ranges {751039=582900,1776135=174904} 2014-07-01 17:02:15,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: writing mira0437250-137 from 751039 to 1333939 tid 1 2014-07-01 17:02:15,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: writing mira0437250-137 from 1776135 to 1951039 tid 2 2014-07-01 17:02:15,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: write oid 382 current snap is 189 2014-07-01 17:02:15,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: seq_num 993 ranges {447293=706128,1647292=1} 2014-07-01 17:02:15,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: writing mira0437250-382 from 447293 to 1153421 tid 1 2014-07-01 17:02:15,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: writing mira0437250-382 from 1647292 to 1647293 tid 2 2014-07-01 17:02:15,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: write oid 130 current snap is 189 2014-07-01 17:02:15,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: seq_num 994 ranges {791098=678956,2120751=270347} 2014-07-01 17:02:15,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: writing mira0437250-130 from 791098 to 1470054 tid 1 2014-07-01 17:02:15,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: writing mira0437250-130 from 2120751 to 2391098 tid 2 2014-07-01 17:02:15,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: read oid 436 snap 175 2014-07-01 17:02:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 1157 (ObjNum 115 snap 0 seq_num 115) dirty exists 2014-07-01 17:02:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 1 to mira0437250-137 2014-07-01 17:02:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 2 to mira0437250-137 2014-07-01 17:02:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 4 to mira0437250-137 2014-07-01 17:02:15,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: finishing write tid 5 to mira0437250-137 2014-07-01 17:02:15,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 1161 (ObjNum 992 snap 189 seq_num 992) dirty exists 2014-07-01 17:02:15,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: finishing write tid 1 to mira0437250-382 2014-07-01 17:02:15,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: finishing write tid 2 to mira0437250-382 2014-07-01 17:02:15,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: finishing write tid 4 to mira0437250-382 2014-07-01 17:02:15,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: finishing write tid 5 to mira0437250-382 2014-07-01 17:02:15,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 564 (ObjNum 993 snap 189 seq_num 993) dirty exists 2014-07-01 17:02:15,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1847: done (4 left) 2014-07-01 17:02:15,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1848: done (3 left) 2014-07-01 17:02:15,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1849: done (2 left) 2014-07-01 17:02:15,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: read oid 136 snap -1 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1853: rollback oid 370 current snap is 189 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 370 to 178 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: snap_create 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 0 (ObjNum 39408864 snap 0 seq_num 37059680) dirty dne 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: finishing write tid 1 to mira0437250-130 2014-07-01 17:02:15,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: expect (ObjNum 435 snap 0 seq_num 435) 2014-07-01 17:02:15,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: expect (ObjNum 135 snap 0 seq_num 135) 2014-07-01 17:02:16,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: finishing write tid 2 to mira0437250-130 2014-07-01 17:02:16,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: finishing write tid 4 to mira0437250-130 2014-07-01 17:02:16,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: finishing write tid 5 to mira0437250-130 2014-07-01 17:02:16,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 611 (ObjNum 994 snap 189 seq_num 994) dirty exists 2014-07-01 17:02:16,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:1850: done (4 left) 2014-07-01 17:02:16,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1851: done (3 left) 2014-07-01 17:02:16,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1852: done (2 left) 2014-07-01 17:02:16,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1853: done (1 left) 2014-07-01 17:02:16,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1854: done (0 left) 2014-07-01 17:02:16,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: snap_remove snap 178 2014-07-01 17:02:16,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:1855: done (0 left) 2014-07-01 17:02:16,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:1856: snap_create 2014-07-01 17:02:17,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1856: done (0 left) 2014-07-01 17:02:17,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:1857: snap_remove snap 183 2014-07-01 17:02:17,979.979 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 34' 2014-07-01 17:02:18,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1857: done (0 left) 2014-07-01 17:02:18,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:1858: delete oid 197 current snap is 191 2014-07-01 17:02:18,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1858: done (0 left) 2014-07-01 17:02:18,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1859: read oid 19 snap -1 2014-07-01 17:02:18,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: snap_remove snap 186 2014-07-01 17:02:18,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:1859: expect (ObjNum 657 snap 73 seq_num 657) 2014-07-01 17:02:19,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:1859: done (1 left) 2014-07-01 17:02:19,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1860: done (0 left) 2014-07-01 17:02:19,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: read oid 312 snap 175 2014-07-01 17:02:19,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: read oid 313 snap 179 2014-07-01 17:02:19,650.650 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 34 2014-07-01 17:02:19,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: snap_remove snap 182 2014-07-01 17:02:19,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: expect (ObjNum 701 snap 92 seq_num 701) 2014-07-01 17:02:20,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: expect (ObjNum 312 snap 0 seq_num 312) 2014-07-01 17:02:20,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:1861: done (2 left) 2014-07-01 17:02:20,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1862: done (1 left) 2014-07-01 17:02:20,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1863: done (0 left) 2014-07-01 17:02:20,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:1864: snap_create 2014-07-01 17:02:21,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1864: done (0 left) 2014-07-01 17:02:21,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: read oid 161 snap -1 2014-07-01 17:02:21,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1866: snap_remove snap 140 2014-07-01 17:02:21,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: expect (ObjNum 844 snap 141 seq_num 844) 2014-07-01 17:02:22,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: expect (ObjNum 1717 snap 0 seq_num 1717) 2014-07-01 17:02:22,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1865: done (1 left) 2014-07-01 17:02:22,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1866: done (0 left) 2014-07-01 17:02:22,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:1867: rollback oid 401 current snap is 192 2014-07-01 17:02:22,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 401 to 156 2014-07-01 17:02:22,618.618 INFO:teuthology.task.rados.rados.0.mira043.stdout:1868: snap_create 2014-07-01 17:02:22,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 904 (ObjNum 400 snap 0 seq_num 400) dirty exists 2014-07-01 17:02:23,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3105: done (6 left) 2014-07-01 17:02:23,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3107: done (5 left) 2014-07-01 17:02:23,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: done (4 left) 2014-07-01 17:02:23,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: done (3 left) 2014-07-01 17:02:23,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: done (2 left) 2014-07-01 17:02:23,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: done (1 left) 2014-07-01 17:02:23,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: done (0 left) 2014-07-01 17:02:23,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3114: read oid 240 snap -1 2014-07-01 17:02:23,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: read oid 118 snap -1 2014-07-01 17:02:23,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3116: read oid 355 snap -1 2014-07-01 17:02:23,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3117: read oid 85 snap -1 2014-07-01 17:02:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 29 current snap is 0 2014-07-01 17:02:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: seq_num 1939 ranges {3506176=454656,3960832=401408} 2014-07-01 17:02:23,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: writing mira0435601-29 from 3506176 to 3960832 tid 1 2014-07-01 17:02:23,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: writing mira0435601-29 from 3960832 to 4362240 tid 2 2014-07-01 17:02:23,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 421 current snap is 0 2014-07-01 17:02:23,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: seq_num 1940 ranges {2707456=483328,3190784=659456,3850240=507904,4358144=438272} 2014-07-01 17:02:23,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: writing mira0435601-421 from 2707456 to 3190784 tid 1 2014-07-01 17:02:23,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: writing mira0435601-421 from 3190784 to 3850240 tid 2 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: writing mira0435601-421 from 3850240 to 4358144 tid 3 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: writing mira0435601-421 from 4358144 to 4796416 tid 4 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 84 current snap is 0 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: seq_num 1941 ranges {2936832=757760,3694592=643072,4337664=561152} 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1867: done (1 left) 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1868: done (0 left) 2014-07-01 17:02:23,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: write oid 123 current snap is 193 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: seq_num 996 ranges {430134=657637,1794520=587476,2830133=1} 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: writing mira0437250-123 from 430134 to 1087771 tid 1 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: writing mira0437250-123 from 1794520 to 2381996 tid 2 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: writing mira0437250-123 from 2830133 to 2830134 tid 3 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: delete oid 240 current snap is 193 2014-07-01 17:02:23,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0435601-84 from 2936832 to 3694592 tid 1 2014-07-01 17:02:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: finishing write tid 1 to mira0437250-123 2014-07-01 17:02:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: finishing write tid 2 to mira0437250-123 2014-07-01 17:02:23,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: finishing write tid 3 to mira0437250-123 2014-07-01 17:02:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: finishing write tid 5 to mira0437250-123 2014-07-01 17:02:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1870: done (1 left) 2014-07-01 17:02:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: write oid 351 current snap is 193 2014-07-01 17:02:23,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: seq_num 998 ranges {587569=753694,2105088=727866,3618974=168595} 2014-07-01 17:02:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: writing mira0437250-351 from 587569 to 1341263 tid 1 2014-07-01 17:02:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: writing mira0437250-351 from 2105088 to 2832954 tid 2 2014-07-01 17:02:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: writing mira0437250-351 from 3618974 to 3787569 tid 3 2014-07-01 17:02:23,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0435601-84 from 3694592 to 4337664 tid 2 2014-07-01 17:02:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0435601-84 from 4337664 to 4898816 tid 3 2014-07-01 17:02:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 130 current snap is 0 2014-07-01 17:02:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: seq_num 1942 ranges {1011712=626688,1638400=475136,2113536=454656,2568192=274432} 2014-07-01 17:02:23,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: writing mira0435601-130 from 1011712 to 1638400 tid 1 2014-07-01 17:02:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: writing mira0435601-130 from 1638400 to 2113536 tid 2 2014-07-01 17:02:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: writing mira0435601-130 from 2113536 to 2568192 tid 3 2014-07-01 17:02:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: writing mira0435601-130 from 2568192 to 2842624 tid 4 2014-07-01 17:02:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 17:02:23,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: seq_num 1943 ranges {2822144=147456} 2014-07-01 17:02:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: writing mira0435601-336 from 2822144 to 2969600 tid 1 2014-07-01 17:02:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: read oid 250 snap -1 2014-07-01 17:02:23,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3114: expect (ObjNum 1922 snap 0 seq_num 1922) 2014-07-01 17:02:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1872: delete oid 327 current snap is 193 2014-07-01 17:02:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: finishing write tid 6 to mira0437250-123 2014-07-01 17:02:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 789 (ObjNum 996 snap 193 seq_num 996) dirty exists 2014-07-01 17:02:24,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: finishing write tid 1 to mira0437250-351 2014-07-01 17:02:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: finishing write tid 2 to mira0437250-351 2014-07-01 17:02:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: finishing write tid 3 to mira0437250-351 2014-07-01 17:02:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: finishing write tid 5 to mira0437250-351 2014-07-01 17:02:24,023.023 INFO:teuthology.task.rados.rados.0.mira043.stdout:1869: done (2 left) 2014-07-01 17:02:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1872: done (1 left) 2014-07-01 17:02:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: read oid 65 snap 181 2014-07-01 17:02:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: read oid 160 snap -1 2014-07-01 17:02:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: finishing write tid 6 to mira0437250-351 2014-07-01 17:02:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 461 (ObjNum 998 snap 193 seq_num 998) dirty exists 2014-07-01 17:02:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1871: done (2 left) 2014-07-01 17:02:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1875: read oid 331 snap -1 2014-07-01 17:02:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: delete oid 317 current snap is 193 2014-07-01 17:02:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: expect (ObjNum 760 snap 111 seq_num 760) 2014-07-01 17:02:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1873: done (3 left) 2014-07-01 17:02:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1876: done (2 left) 2014-07-01 17:02:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: snap_remove snap 177 2014-07-01 17:02:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1875: expect (ObjNum 330 snap 0 seq_num 330) 2014-07-01 17:02:24,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: expect (ObjNum 610 snap 47 seq_num 610) 2014-07-01 17:02:24,660.660 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:24,660.660 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:24,661.661 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:02:24,661.661 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:02:24,661.661 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:02:24,661.661 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:02:25,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: expect (ObjNum 117 snap 0 seq_num 117) 2014-07-01 17:02:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1874: done (2 left) 2014-07-01 17:02:25,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:1875: done (1 left) 2014-07-01 17:02:25,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1877: done (0 left) 2014-07-01 17:02:25,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: write oid 277 current snap is 193 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: seq_num 1001 ranges {687787=705203,1487786=1} 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: writing mira0437250-277 from 687787 to 1392990 tid 1 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: writing mira0437250-277 from 1487786 to 1487787 tid 2 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: write oid 287 current snap is 193 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: seq_num 1002 ranges {632616=677007,2071248=161368} 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: writing mira0437250-287 from 632616 to 1309623 tid 1 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: writing mira0437250-287 from 2071248 to 2232616 tid 2 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1880: rollback oid 426 current snap is 193 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 426 to 179 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 1 to mira0437250-277 2014-07-01 17:02:26,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 2 to mira0437250-277 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 4 to mira0437250-277 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: finishing write tid 5 to mira0437250-277 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 586 (ObjNum 1001 snap 193 seq_num 1001) dirty exists 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1878: done (2 left) 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: read oid 83 snap -1 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1882: snap_create 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: finishing write tid 1 to mira0437250-287 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 426 v 0 (ObjNum 17667536 snap 0 seq_num 38731728) dirty dne 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: finishing write tid 2 to mira0437250-287 2014-07-01 17:02:26,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: expect (ObjNum 82 snap 0 seq_num 82) 2014-07-01 17:02:26,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: finishing write tid 4 to mira0437250-287 2014-07-01 17:02:26,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: finishing write tid 5 to mira0437250-287 2014-07-01 17:02:26,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 662 (ObjNum 1002 snap 193 seq_num 1002) dirty exists 2014-07-01 17:02:26,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3116: expect (ObjNum 754 snap 0 seq_num 754) 2014-07-01 17:02:26,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1879: done (3 left) 2014-07-01 17:02:26,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1880: done (2 left) 2014-07-01 17:02:26,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1881: done (1 left) 2014-07-01 17:02:26,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:1882: done (0 left) 2014-07-01 17:02:26,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: write oid 24 current snap is 194 2014-07-01 17:02:26,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: seq_num 1003 ranges {577446=400235,1501732=616762,2605900=371546} 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: writing mira0437250-24 from 577446 to 977681 tid 1 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: writing mira0437250-24 from 1501732 to 2118494 tid 2 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: writing mira0437250-24 from 2605900 to 2977446 tid 3 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1884: rollback oid 27 current snap is 194 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 189 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 1 to mira0437250-24 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 2 to mira0437250-24 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: read oid 489 snap -1 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: read oid 459 snap -1 2014-07-01 17:02:26,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1887: rollback oid 266 current snap is 194 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 266 to 175 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: write oid 480 current snap is 194 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: seq_num 1004 ranges {671652=655688,1896441=535533,2671651=1} 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: writing mira0437250-480 from 671652 to 1327340 tid 1 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: writing mira0437250-480 from 1896441 to 2431974 tid 2 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: writing mira0437250-480 from 2671651 to 2671652 tid 3 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1889: snap_create 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: expect (ObjNum 458 snap 0 seq_num 458) 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 3 to mira0437250-24 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 5 to mira0437250-24 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: finishing write tid 6 to mira0437250-24 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 24 v 709 (ObjNum 1003 snap 194 seq_num 1003) dirty exists 2014-07-01 17:02:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 445 (ObjNum 265 snap 0 seq_num 265) dirty exists 2014-07-01 17:02:26,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 828 (ObjNum 647 snap 67 seq_num 647) dirty exists 2014-07-01 17:02:26,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: expect (ObjNum 803 snap 129 seq_num 803) 2014-07-01 17:02:27,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: finishing write tid 1 to mira0437250-480 2014-07-01 17:02:27,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: finishing write tid 2 to mira0437250-480 2014-07-01 17:02:27,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: finishing write tid 3 to mira0437250-480 2014-07-01 17:02:27,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: finishing write tid 5 to mira0437250-480 2014-07-01 17:02:27,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: finishing write tid 6 to mira0437250-480 2014-07-01 17:02:27,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 802 (ObjNum 1004 snap 194 seq_num 1004) dirty exists 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: finishing write tid 1 to mira0435601-29 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: finishing write tid 2 to mira0435601-29 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: finishing write tid 4 to mira0435601-29 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: finishing write tid 5 to mira0435601-29 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 237 (ObjNum 1939 snap 0 seq_num 1939) dirty exists 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 1 to mira0435601-421 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3114: done (9 left) 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: done (8 left) 2014-07-01 17:02:27,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3116: done (7 left) 2014-07-01 17:02:27,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3117: done (6 left) 2014-07-01 17:02:27,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: done (5 left) 2014-07-01 17:02:27,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3124: delete oid 100 current snap is 0 2014-07-01 17:02:27,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 2 to mira0435601-421 2014-07-01 17:02:27,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 3 to mira0435601-421 2014-07-01 17:02:27,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 4 to mira0435601-421 2014-07-01 17:02:27,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 6 to mira0435601-421 2014-07-01 17:02:27,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: finishing write tid 7 to mira0435601-421 2014-07-01 17:02:27,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 308 (ObjNum 1940 snap 0 seq_num 1940) dirty exists 2014-07-01 17:02:27,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 1 to mira0435601-84 2014-07-01 17:02:27,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 2 to mira0435601-84 2014-07-01 17:02:27,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 3 to mira0435601-84 2014-07-01 17:02:27,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 5 to mira0435601-84 2014-07-01 17:02:27,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 1 to mira0435601-130 2014-07-01 17:02:27,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 2 to mira0435601-130 2014-07-01 17:02:27,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 3 to mira0435601-130 2014-07-01 17:02:27,410.410 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 4 to mira0435601-130 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 6 to mira0435601-130 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 6 to mira0435601-84 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 312 (ObjNum 1941 snap 0 seq_num 1941) dirty exists 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: finishing write tid 7 to mira0435601-130 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 265 (ObjNum 1942 snap 0 seq_num 1942) dirty exists 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: finishing write tid 1 to mira0435601-336 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: expect (ObjNum 1871 snap 0 seq_num 1871) 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: finishing write tid 3 to mira0435601-336 2014-07-01 17:02:27,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: finishing write tid 4 to mira0435601-336 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 187 (ObjNum 1943 snap 0 seq_num 1943) dirty exists 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: done (5 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: done (4 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: done (3 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: done (2 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: done (1 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3124: done (0 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3125: read oid 373 snap -1 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3126: delete oid 347 current snap is 0 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3125: done (1 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3126: done (0 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: delete oid 245 current snap is 0 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: done (0 left) 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 17:02:27,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: seq_num 1947 ranges {3739648=749568,4489216=438272,4927488=729088,5656576=438272} 2014-07-01 17:02:27,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0435601-31 from 3739648 to 4489216 tid 1 2014-07-01 17:02:27,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0435601-31 from 4489216 to 4927488 tid 2 2014-07-01 17:02:27,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0435601-31 from 4927488 to 5656576 tid 3 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0435601-31 from 5656576 to 6094848 tid 4 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 287 current snap is 0 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: seq_num 1948 ranges {122880=421888,544768=622592,1167360=446464,1613824=536576} 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: writing mira0435601-287 from 122880 to 544768 tid 1 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: writing mira0435601-287 from 544768 to 1167360 tid 2 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: writing mira0435601-287 from 1167360 to 1613824 tid 3 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: writing mira0435601-287 from 1613824 to 2150400 tid 4 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3130: delete oid 128 current snap is 0 2014-07-01 17:02:27,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 1 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 2 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 3 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 4 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 6 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 7 to mira0435601-31 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 296 (ObjNum 1947 snap 0 seq_num 1947) dirty exists 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 1 to mira0435601-287 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 2 to mira0435601-287 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 3 to mira0435601-287 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: done (2 left) 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3130: done (1 left) 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 196 current snap is 0 2014-07-01 17:02:27,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: seq_num 1950 ranges {0=569344,569344=589824,1159168=499712,1658880=114688} 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0435601-196 from 0 to 569344 tid 1 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0435601-196 from 569344 to 1159168 tid 2 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0435601-196 from 1159168 to 1658880 tid 3 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0435601-196 from 1658880 to 1773568 tid 4 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: read oid 185 snap -1 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 4 to mira0435601-287 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 6 to mira0435601-287 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: finishing write tid 7 to mira0435601-287 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 191 (ObjNum 1948 snap 0 seq_num 1948) dirty exists 2014-07-01 17:02:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 1 to mira0435601-196 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 2 to mira0435601-196 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: done (2 left) 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: read oid 96 snap -1 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3134: read oid 45 snap -1 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3135: read oid 497 snap -1 2014-07-01 17:02:27,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3136: delete oid 285 current snap is 0 2014-07-01 17:02:27,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 3 to mira0435601-196 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 4 to mira0435601-196 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 6 to mira0435601-196 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3136: done (5 left) 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 363 current snap is 0 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: seq_num 1952 ranges {0=761856,761856=602112,1363968=446464,1810432=557056} 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: writing mira0435601-363 from 0 to 761856 tid 1 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: writing mira0435601-363 from 761856 to 1363968 tid 2 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: writing mira0435601-363 from 1363968 to 1810432 tid 3 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: writing mira0435601-363 from 1810432 to 2367488 tid 4 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 36 current snap is 0 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: seq_num 1953 ranges {3760128=790528,4550656=401408} 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1883: done (6 left) 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1884: done (5 left) 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1885: done (4 left) 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1886: done (3 left) 2014-07-01 17:02:27,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:1887: done (2 left) 2014-07-01 17:02:27,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1888: done (1 left) 2014-07-01 17:02:27,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1889: done (0 left) 2014-07-01 17:02:27,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:1890: snap_create 2014-07-01 17:02:27,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: writing mira0435601-36 from 3760128 to 4550656 tid 1 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: writing mira0435601-36 from 4550656 to 4952064 tid 2 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 403 current snap is 0 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: seq_num 1954 ranges {0=12288} 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: writing mira0435601-403 from 0 to 12288 tid 1 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 305 current snap is 0 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: seq_num 1955 ranges {0=532480,532480=401408} 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: writing mira0435601-305 from 0 to 532480 tid 1 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: writing mira0435601-305 from 532480 to 933888 tid 2 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 177 current snap is 0 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: seq_num 1956 ranges {6193152=557056,6750208=630784,7380992=495616,7876608=479232} 2014-07-01 17:02:27,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: writing mira0435601-177 from 6193152 to 6750208 tid 1 2014-07-01 17:02:27,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: writing mira0435601-177 from 6750208 to 7380992 tid 2 2014-07-01 17:02:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: writing mira0435601-177 from 7380992 to 7876608 tid 3 2014-07-01 17:02:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: writing mira0435601-177 from 7876608 to 8355840 tid 4 2014-07-01 17:02:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 17:02:27,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: seq_num 1957 ranges {479232=634880,1114112=409600,1523712=569344,2093056=225280} 2014-07-01 17:02:27,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0435601-186 from 479232 to 1114112 tid 1 2014-07-01 17:02:27,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0435601-186 from 1114112 to 1523712 tid 2 2014-07-01 17:02:27,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0435601-186 from 1523712 to 2093056 tid 3 2014-07-01 17:02:28,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0435601-186 from 2093056 to 2318336 tid 4 2014-07-01 17:02:28,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3143: read oid 454 snap -1 2014-07-01 17:02:28,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 7 to mira0435601-196 2014-07-01 17:02:28,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 347 (ObjNum 1950 snap 0 seq_num 1950) dirty exists 2014-07-01 17:02:28,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: expect (ObjNum 1847 snap 0 seq_num 1847) 2014-07-01 17:02:28,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3135: expect (ObjNum 1765 snap 0 seq_num 1765) 2014-07-01 17:02:28,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1890: done (0 left) 2014-07-01 17:02:28,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: read oid 91 snap -1 2014-07-01 17:02:28,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:1892: delete oid 255 current snap is 196 2014-07-01 17:02:28,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:1892: done (1 left) 2014-07-01 17:02:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: read oid 202 snap -1 2014-07-01 17:02:28,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: expect (ObjNum 90 snap 0 seq_num 90) 2014-07-01 17:02:29,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 1 to mira0435601-36 2014-07-01 17:02:29,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: expect (ObjNum 940 snap 0 seq_num 940) 2014-07-01 17:02:29,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:1891: done (1 left) 2014-07-01 17:02:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1893: done (0 left) 2014-07-01 17:02:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: write oid 312 current snap is 196 2014-07-01 17:02:29,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: seq_num 1006 ranges {676638=1} 2014-07-01 17:02:29,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: writing mira0437250-312 from 676638 to 676639 tid 1 2014-07-01 17:02:30,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: write oid 442 current snap is 196 2014-07-01 17:02:30,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: seq_num 1007 ranges {372892=1} 2014-07-01 17:02:30,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: writing mira0437250-442 from 372892 to 372893 tid 1 2014-07-01 17:02:30,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: read oid 340 snap -1 2014-07-01 17:02:30,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: read oid 295 snap -1 2014-07-01 17:02:30,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1898: read oid 499 snap -1 2014-07-01 17:02:30,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1899: read oid 57 snap -1 2014-07-01 17:02:30,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:1900: rollback oid 259 current snap is 196 2014-07-01 17:02:30,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 259 to 149 2014-07-01 17:02:30,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: write oid 11 current snap is 196 2014-07-01 17:02:30,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: seq_num 1008 ranges {69146=1} 2014-07-01 17:02:30,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: writing mira0437250-11 from 69146 to 69147 tid 1 2014-07-01 17:02:30,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: snap_create 2014-07-01 17:02:30,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: finishing write tid 1 to mira0437250-312 2014-07-01 17:02:30,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: finishing write tid 3 to mira0437250-312 2014-07-01 17:02:30,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 1 to mira0437250-442 2014-07-01 17:02:30,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 582 (ObjNum 258 snap 0 seq_num 258) dirty exists 2014-07-01 17:02:30,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: finishing write tid 1 to mira0437250-11 2014-07-01 17:02:30,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: finishing write tid 3 to mira0437250-11 2014-07-01 17:02:30,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: finishing write tid 4 to mira0437250-11 2014-07-01 17:02:30,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 782 (ObjNum 1008 snap 196 seq_num 1008) dirty exists 2014-07-01 17:02:30,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 3 to mira0437250-442 2014-07-01 17:02:30,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: finishing write tid 4 to mira0437250-442 2014-07-01 17:02:30,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 442 v 828 (ObjNum 1007 snap 196 seq_num 1007) dirty exists 2014-07-01 17:02:30,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1899: expect (ObjNum 955 snap 175 seq_num 955) 2014-07-01 17:02:30,097.097 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:30,097.097 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:30,097.097 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 17:02:30,097.097 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:02:30,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: finishing write tid 4 to mira0437250-312 2014-07-01 17:02:30,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 1152 (ObjNum 1006 snap 196 seq_num 1006) dirty exists 2014-07-01 17:02:30,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: expect (ObjNum 626 snap 53 seq_num 626) 2014-07-01 17:02:30,888.888 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:02:31,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1894: done (8 left) 2014-07-01 17:02:31,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1895: done (7 left) 2014-07-01 17:02:31,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1896: done (6 left) 2014-07-01 17:02:31,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:1897: done (5 left) 2014-07-01 17:02:31,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:1898: done (4 left) 2014-07-01 17:02:31,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:1899: done (3 left) 2014-07-01 17:02:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1900: done (2 left) 2014-07-01 17:02:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1901: done (1 left) 2014-07-01 17:02:31,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:1902: done (0 left) 2014-07-01 17:02:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1903: rollback oid 106 current snap is 197 2014-07-01 17:02:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 106 to 190 2014-07-01 17:02:31,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:1904: rollback oid 371 current snap is 197 2014-07-01 17:02:31,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 371 to 187 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: read oid 242 snap 187 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: read oid 479 snap 190 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: write oid 389 current snap is 197 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: seq_num 1009 ranges {661528=454677,1846556=14972} 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: writing mira0437250-389 from 661528 to 1116205 tid 1 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: writing mira0437250-389 from 1846556 to 1861528 tid 2 2014-07-01 17:02:31,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: read oid 480 snap -1 2014-07-01 17:02:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 1199 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 17:02:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 1112 (ObjNum 863 snap 145 seq_num 863) dirty exists 2014-07-01 17:02:31,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: expect (ObjNum 591 snap 39 seq_num 591) 2014-07-01 17:02:31,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: expect (ObjNum 241 snap 0 seq_num 241) 2014-07-01 17:02:32,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: expect (ObjNum 1004 snap 194 seq_num 1004) 2014-07-01 17:02:33,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 1 to mira0437250-389 2014-07-01 17:02:33,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 2 to mira0437250-389 2014-07-01 17:02:33,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 4 to mira0437250-389 2014-07-01 17:02:33,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: finishing write tid 5 to mira0437250-389 2014-07-01 17:02:33,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 972 (ObjNum 1009 snap 197 seq_num 1009) dirty exists 2014-07-01 17:02:33,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1903: done (5 left) 2014-07-01 17:02:33,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1904: done (4 left) 2014-07-01 17:02:33,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:1905: done (3 left) 2014-07-01 17:02:33,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:1906: done (2 left) 2014-07-01 17:02:33,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:1907: done (1 left) 2014-07-01 17:02:33,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:1908: done (0 left) 2014-07-01 17:02:33,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: read oid 284 snap -1 2014-07-01 17:02:33,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:1910: read oid 138 snap -1 2014-07-01 17:02:33,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:1911: read oid 426 snap -1 2014-07-01 17:02:33,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: write oid 283 current snap is 197 2014-07-01 17:02:33,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: seq_num 1010 ranges {474134=675058,1925618=148516} 2014-07-01 17:02:33,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: writing mira0437250-283 from 474134 to 1149192 tid 1 2014-07-01 17:02:33,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: writing mira0437250-283 from 1925618 to 2074134 tid 2 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: write oid 144 current snap is 197 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: seq_num 1011 ranges {569364=798286,1769363=1} 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: writing mira0437250-144 from 569364 to 1367650 tid 1 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: writing mira0437250-144 from 1769363 to 1769364 tid 2 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: read oid 365 snap -1 2014-07-01 17:02:33,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: expect (ObjNum 781 snap 124 seq_num 781) 2014-07-01 17:02:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 1 to mira0435601-363 2014-07-01 17:02:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 2 to mira0435601-363 2014-07-01 17:02:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: done (11 left) 2014-07-01 17:02:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: done (10 left) 2014-07-01 17:02:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: done (9 left) 2014-07-01 17:02:34,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:3135: done (8 left) 2014-07-01 17:02:34,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 17:02:34,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: seq_num 1958 ranges {3022848=352256} 2014-07-01 17:02:34,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: writing mira0435601-208 from 3022848 to 3375104 tid 1 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3145: read oid 380 snap -1 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 3 to mira0435601-363 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 2 to mira0435601-36 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 4 to mira0435601-36 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: finishing write tid 1 to mira0435601-403 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: finishing write tid 3 to mira0435601-403 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 4 to mira0435601-363 2014-07-01 17:02:34,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 6 to mira0435601-363 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: finishing write tid 7 to mira0435601-363 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 301 (ObjNum 1952 snap 0 seq_num 1952) dirty exists 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: finishing write tid 1 to mira0435601-305 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: finishing write tid 2 to mira0435601-305 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: done (9 left) 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: read oid 7 snap -1 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3147: read oid 273 snap -1 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: finishing write tid 4 to mira0435601-305 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 1 to mira0435601-177 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 2 to mira0435601-177 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 3 to mira0435601-177 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 4 to mira0435601-177 2014-07-01 17:02:34,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 6 to mira0435601-177 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 144 current snap is 0 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: seq_num 1959 ranges {2109440=778240,2887680=626688,3514368=176128} 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: writing mira0435601-144 from 2109440 to 2887680 tid 1 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: writing mira0435601-144 from 2887680 to 3514368 tid 2 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: writing mira0435601-144 from 3514368 to 3690496 tid 3 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3149: read oid 428 snap -1 2014-07-01 17:02:34,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3134: expect (ObjNum 1108 snap 0 seq_num 1108) 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: finishing write tid 1 to mira0437250-283 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: finishing write tid 2 to mira0437250-283 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: finishing write tid 4 to mira0437250-283 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: finishing write tid 5 to mira0437250-283 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 283 v 786 (ObjNum 1010 snap 197 seq_num 1010) dirty exists 2014-07-01 17:02:34,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: expect (ObjNum 850 snap 141 seq_num 850) 2014-07-01 17:02:34,403.403 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:02:34,403.403 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pg_num 44' 2014-07-01 17:02:34,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:1910: expect (ObjNum 137 snap 0 seq_num 137) 2014-07-01 17:02:35,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 1 to mira0437250-144 2014-07-01 17:02:35,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 2 to mira0437250-144 2014-07-01 17:02:35,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 4 to mira0437250-144 2014-07-01 17:02:35,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: finishing write tid 5 to mira0437250-144 2014-07-01 17:02:35,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 1115 (ObjNum 1011 snap 197 seq_num 1011) dirty exists 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1909: done (5 left) 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1910: done (4 left) 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1911: done (3 left) 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1912: done (2 left) 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1913: done (1 left) 2014-07-01 17:02:35,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:1914: done (0 left) 2014-07-01 17:02:35,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:1915: delete oid 117 current snap is 197 2014-07-01 17:02:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1915: done (0 left) 2014-07-01 17:02:35,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: write oid 181 current snap is 197 2014-07-01 17:02:35,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: seq_num 1013 ranges {457999=777991,1830894=455610,2695083=562916} 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: writing mira0437250-181 from 457999 to 1235990 tid 1 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: writing mira0437250-181 from 1830894 to 2286504 tid 2 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: writing mira0437250-181 from 2695083 to 3257999 tid 3 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: rollback oid 320 current snap is 197 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 320 to 174 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 1 to mira0437250-181 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 2 to mira0437250-181 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1918: delete oid 12 current snap is 197 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 602 (ObjNum 319 snap 0 seq_num 319) dirty exists 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1917: done (2 left) 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1918: done (1 left) 2014-07-01 17:02:35,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:1919: snap_create 2014-07-01 17:02:35,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 3 to mira0437250-181 2014-07-01 17:02:35,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 5 to mira0437250-181 2014-07-01 17:02:35,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: finishing write tid 6 to mira0437250-181 2014-07-01 17:02:35,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 181 v 917 (ObjNum 1013 snap 197 seq_num 1013) dirty exists 2014-07-01 17:02:35,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:1916: done (1 left) 2014-07-01 17:02:35,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1919: done (0 left) 2014-07-01 17:02:35,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:1920: rollback oid 277 current snap is 198 2014-07-01 17:02:35,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 277 to 184 2014-07-01 17:02:35,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:1921: snap_create 2014-07-01 17:02:35,533.533 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pg_num to 44 2014-07-01 17:02:35,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 588 (ObjNum 845 snap 141 seq_num 845) dirty exists 2014-07-01 17:02:36,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1920: done (1 left) 2014-07-01 17:02:36,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1921: done (0 left) 2014-07-01 17:02:36,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1922: rollback oid 416 current snap is 199 2014-07-01 17:02:36,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 416 to 187 2014-07-01 17:02:36,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: snap_remove snap 193 2014-07-01 17:02:36,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:02:37,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1922: done (1 left) 2014-07-01 17:02:37,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1923: done (0 left) 2014-07-01 17:02:37,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: write oid 95 current snap is 199 2014-07-01 17:02:37,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: seq_num 1015 ranges {462987=474047,1262986=1} 2014-07-01 17:02:37,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: writing mira0437250-95 from 462987 to 937034 tid 1 2014-07-01 17:02:37,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: writing mira0437250-95 from 1262986 to 1262987 tid 2 2014-07-01 17:02:37,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: read oid 167 snap -1 2014-07-01 17:02:37,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: read oid 445 snap -1 2014-07-01 17:02:37,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: write oid 245 current snap is 199 2014-07-01 17:02:37,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: seq_num 1016 ranges {558217=400000} 2014-07-01 17:02:37,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: writing mira0437250-245 from 558217 to 958217 tid 1 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: write oid 131 current snap is 199 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: seq_num 1017 ranges {503046=452728,1548478=154568} 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: writing mira0437250-131 from 503046 to 955774 tid 1 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: writing mira0437250-131 from 1548478 to 1703046 tid 2 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: read oid 193 snap -1 2014-07-01 17:02:37,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: expect (ObjNum 166 snap 0 seq_num 166) 2014-07-01 17:02:37,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: finishing write tid 1 to mira0437250-95 2014-07-01 17:02:37,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: finishing write tid 2 to mira0437250-95 2014-07-01 17:02:37,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: finishing write tid 4 to mira0437250-95 2014-07-01 17:02:37,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: expect (ObjNum 444 snap 0 seq_num 444) 2014-07-01 17:02:38,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: finishing write tid 5 to mira0437250-95 2014-07-01 17:02:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 545 (ObjNum 1015 snap 199 seq_num 1015) dirty exists 2014-07-01 17:02:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: finishing write tid 1 to mira0437250-131 2014-07-01 17:02:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: finishing write tid 2 to mira0437250-131 2014-07-01 17:02:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: finishing write tid 4 to mira0437250-131 2014-07-01 17:02:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: expect (ObjNum 192 snap 0 seq_num 192) 2014-07-01 17:02:38,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: finishing write tid 5 to mira0435601-305 2014-07-01 17:02:38,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 367 (ObjNum 1955 snap 0 seq_num 1955) dirty exists 2014-07-01 17:02:38,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 1 to mira0435601-186 2014-07-01 17:02:38,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 2 to mira0435601-186 2014-07-01 17:02:38,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3134: done (12 left) 2014-07-01 17:02:38,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: done (11 left) 2014-07-01 17:02:38,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3150: read oid 311 snap -1 2014-07-01 17:02:38,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 3 to mira0435601-186 2014-07-01 17:02:38,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3143: expect (ObjNum 1920 snap 0 seq_num 1920) 2014-07-01 17:02:38,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: finishing write tid 1 to mira0437250-245 2014-07-01 17:02:38,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: finishing write tid 3 to mira0437250-245 2014-07-01 17:02:38,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: finishing write tid 5 to mira0437250-131 2014-07-01 17:02:38,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 706 (ObjNum 1017 snap 199 seq_num 1017) dirty exists 2014-07-01 17:02:38,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: finishing write tid 4 to mira0437250-245 2014-07-01 17:02:38,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 245 v 568 (ObjNum 1016 snap 199 seq_num 1016) dirty exists 2014-07-01 17:02:39,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:1924: done (5 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1925: done (4 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1926: done (3 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1927: done (2 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1928: done (1 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1929: done (0 left) 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: write oid 41 current snap is 199 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: seq_num 1018 ranges {447876=424532,1627538=441658,2447875=1} 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: writing mira0437250-41 from 447876 to 872408 tid 1 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: writing mira0437250-41 from 1627538 to 2069196 tid 2 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: writing mira0437250-41 from 2447875 to 2447876 tid 3 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: write oid 19 current snap is 199 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: seq_num 1019 ranges {659458=548785,1608423=663277,2659457=1} 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: writing mira0437250-19 from 659458 to 1208243 tid 1 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: writing mira0437250-19 from 1608423 to 2271700 tid 2 2014-07-01 17:02:39,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: writing mira0437250-19 from 2659457 to 2659458 tid 3 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1932: delete oid 249 current snap is 199 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: finishing write tid 1 to mira0437250-41 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: finishing write tid 2 to mira0437250-41 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: finishing write tid 3 to mira0437250-41 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: finishing write tid 5 to mira0437250-41 2014-07-01 17:02:39,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: finishing write tid 6 to mira0437250-41 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 761 (ObjNum 1018 snap 199 seq_num 1018) dirty exists 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: finishing write tid 1 to mira0437250-19 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: finishing write tid 2 to mira0437250-19 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: finishing write tid 3 to mira0437250-19 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: finishing write tid 5 to mira0437250-19 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1930: done (2 left) 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1932: done (1 left) 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: delete oid 238 current snap is 199 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: finishing write tid 6 to mira0437250-19 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 701 (ObjNum 1019 snap 199 seq_num 1019) dirty exists 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1931: done (1 left) 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1933: done (0 left) 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: delete oid 53 current snap is 199 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1934: done (0 left) 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: read oid 142 snap -1 2014-07-01 17:02:39,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:1936: snap_create 2014-07-01 17:02:39,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 17:02:39,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1935: done (1 left) 2014-07-01 17:02:39,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1936: done (0 left) 2014-07-01 17:02:39,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1937: read oid 134 snap -1 2014-07-01 17:02:39,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:1938: snap_create 2014-07-01 17:02:39,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:1937: expect (ObjNum 665 snap 75 seq_num 665) 2014-07-01 17:02:40,549.549 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:40,549.549 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:40,550.550 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 17:02:40,550.550 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:02:40,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1937: done (1 left) 2014-07-01 17:02:40,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:1938: done (0 left) 2014-07-01 17:02:40,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1939: rollback oid 349 current snap is 201 2014-07-01 17:02:40,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 349 to 191 2014-07-01 17:02:40,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: read oid 202 snap -1 2014-07-01 17:02:40,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: write oid 63 current snap is 201 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: seq_num 1023 ranges {588152=1} 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: writing mira0437250-63 from 588152 to 588153 tid 1 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: write oid 359 current snap is 201 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: seq_num 1024 ranges {799733=1} 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: writing mira0437250-359 from 799733 to 799734 tid 1 2014-07-01 17:02:40,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: write oid 37 current snap is 201 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: seq_num 1025 ranges {744564=451803,1544563=1} 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: writing mira0437250-37 from 744564 to 1196367 tid 1 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: writing mira0437250-37 from 1544563 to 1544564 tid 2 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: write oid 316 current snap is 201 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: seq_num 1026 ranges {688370=424631,1730483=557887} 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: writing mira0437250-316 from 688370 to 1113001 tid 1 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: writing mira0437250-316 from 1730483 to 2288370 tid 2 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: delete oid 203 current snap is 201 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 1 to mira0437250-63 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 3 to mira0437250-63 2014-07-01 17:02:40,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: finishing write tid 4 to mira0437250-63 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 737 (ObjNum 1023 snap 201 seq_num 1023) dirty exists 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 1119 (ObjNum 697 snap 90 seq_num 697) dirty exists 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: finishing write tid 1 to mira0437250-37 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 1 to mira0437250-359 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: finishing write tid 2 to mira0437250-37 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: finishing write tid 4 to mira0437250-37 2014-07-01 17:02:40,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 3 to mira0437250-359 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: finishing write tid 4 to mira0437250-359 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 764 (ObjNum 1024 snap 201 seq_num 1024) dirty exists 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 1 to mira0437250-316 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: finishing write tid 5 to mira0437250-37 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 831 (ObjNum 1025 snap 201 seq_num 1025) dirty exists 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1939: done (6 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1940: done (5 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1941: done (4 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1942: done (3 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1943: done (2 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1945: done (1 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: delete oid 294 current snap is 201 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1946: done (1 left) 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: write oid 121 current snap is 201 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: seq_num 1029 ranges {788587=526540,1883312=503149,2835521=451956,3988586=1} 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: writing mira0437250-121 from 788587 to 1315127 tid 1 2014-07-01 17:02:40,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: writing mira0437250-121 from 1883312 to 2386461 tid 2 2014-07-01 17:02:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: writing mira0437250-121 from 2835521 to 3287477 tid 3 2014-07-01 17:02:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: writing mira0437250-121 from 3988586 to 3988587 tid 4 2014-07-01 17:02:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: write oid 187 current snap is 201 2014-07-01 17:02:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: seq_num 1030 ranges {484841=499369,1598469=724768,2790219=586298,3684840=1} 2014-07-01 17:02:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: writing mira0437250-187 from 484841 to 984210 tid 1 2014-07-01 17:02:40,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: writing mira0437250-187 from 1598469 to 2323237 tid 2 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: writing mira0437250-187 from 2790219 to 3376517 tid 3 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: writing mira0437250-187 from 3684840 to 3684841 tid 4 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: write oid 148 current snap is 201 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: seq_num 1031 ranges {428646=1} 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: writing mira0437250-148 from 428646 to 428647 tid 1 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: read oid 112 snap -1 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 2 to mira0437250-316 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 4 to mira0437250-316 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: finishing write tid 5 to mira0437250-316 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 559 (ObjNum 1026 snap 201 seq_num 1026) dirty exists 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 1 to mira0437250-121 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 2 to mira0437250-121 2014-07-01 17:02:40,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 3 to mira0437250-121 2014-07-01 17:02:40,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 4 to mira0437250-121 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 6 to mira0437250-121 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: finishing write tid 7 to mira0437250-121 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 813 (ObjNum 1029 snap 201 seq_num 1029) dirty exists 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 1 to mira0437250-187 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 2 to mira0437250-187 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1944: done (4 left) 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1947: done (3 left) 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: write oid 129 current snap is 201 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: seq_num 1032 ranges {773476=400000} 2014-07-01 17:02:40,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: writing mira0437250-129 from 773476 to 1173476 tid 1 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: rollback oid 322 current snap is 201 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 322 to 174 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 3 to mira0437250-187 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 4 to mira0437250-187 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 6 to mira0437250-187 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: finishing write tid 7 to mira0437250-187 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 1160 (ObjNum 1030 snap 201 seq_num 1030) dirty exists 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1948: done (4 left) 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: write oid 281 current snap is 201 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: seq_num 1033 ranges {586082=450878,1386081=1} 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: writing mira0437250-281 from 586082 to 1036960 tid 1 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: writing mira0437250-281 from 1386081 to 1386082 tid 2 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1954: delete oid 332 current snap is 201 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 1 to mira0437250-148 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 3 to mira0437250-148 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: finishing write tid 4 to mira0437250-148 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 763 (ObjNum 1031 snap 201 seq_num 1031) dirty exists 2014-07-01 17:02:40,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: expect (ObjNum 111 snap 0 seq_num 111) 2014-07-01 17:02:41,316.316 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:02:41,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 826 (ObjNum 691 snap 87 seq_num 691) dirty exists 2014-07-01 17:02:41,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: finishing write tid 1 to mira0437250-129 2014-07-01 17:02:41,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: finishing write tid 3 to mira0437250-129 2014-07-01 17:02:41,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: finishing write tid 4 to mira0437250-129 2014-07-01 17:02:41,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 1122 (ObjNum 1032 snap 201 seq_num 1032) dirty exists 2014-07-01 17:02:41,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: finishing write tid 1 to mira0437250-281 2014-07-01 17:02:41,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: finishing write tid 2 to mira0437250-281 2014-07-01 17:02:41,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: finishing write tid 4 to mira0437250-281 2014-07-01 17:02:41,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: finishing write tid 5 to mira0437250-281 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 817 (ObjNum 1033 snap 201 seq_num 1033) dirty exists 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1949: done (5 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1950: done (4 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1951: done (3 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1952: done (2 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1953: done (1 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1954: done (0 left) 2014-07-01 17:02:41,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: snap_create 2014-07-01 17:02:41,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:1955: done (0 left) 2014-07-01 17:02:41,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: rollback oid 259 current snap is 202 2014-07-01 17:02:41,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 259 to 168 2014-07-01 17:02:41,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:1957: snap_create 2014-07-01 17:02:41,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 586 (ObjNum 917 snap 163 seq_num 917) dirty exists 2014-07-01 17:02:41,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 4 to mira0435601-186 2014-07-01 17:02:41,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 6 to mira0435601-186 2014-07-01 17:02:41,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 5 to mira0435601-36 2014-07-01 17:02:41,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 36 v 359 (ObjNum 1953 snap 0 seq_num 1953) dirty exists 2014-07-01 17:02:41,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: finishing write tid 4 to mira0435601-403 2014-07-01 17:02:41,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 361 (ObjNum 1954 snap 0 seq_num 1954) dirty exists 2014-07-01 17:02:41,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 7 to mira0435601-186 2014-07-01 17:02:41,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 188 (ObjNum 1957 snap 0 seq_num 1957) dirty exists 2014-07-01 17:02:41,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: finishing write tid 7 to mira0435601-177 2014-07-01 17:02:41,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 158 (ObjNum 1956 snap 0 seq_num 1956) dirty exists 2014-07-01 17:02:41,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:3145: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 17:02:42,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:1956: done (1 left) 2014-07-01 17:02:42,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1957: done (0 left) 2014-07-01 17:02:42,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:1958: snap_create 2014-07-01 17:02:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: finishing write tid 1 to mira0435601-208 2014-07-01 17:02:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: finishing write tid 3 to mira0435601-208 2014-07-01 17:02:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: finishing write tid 4 to mira0435601-208 2014-07-01 17:02:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 363 (ObjNum 1958 snap 0 seq_num 1958) dirty exists 2014-07-01 17:02:42,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3147: expect (ObjNum 1884 snap 0 seq_num 1884) 2014-07-01 17:02:43,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:1958: done (0 left) 2014-07-01 17:02:43,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: write oid 330 current snap is 204 2014-07-01 17:02:43,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: seq_num 1035 ranges {474718=545910,1794089=426144,2637177=237541} 2014-07-01 17:02:43,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: writing mira0437250-330 from 474718 to 1020628 tid 1 2014-07-01 17:02:43,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: writing mira0437250-330 from 1794089 to 2220233 tid 2 2014-07-01 17:02:43,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: writing mira0437250-330 from 2637177 to 2874718 tid 3 2014-07-01 17:02:43,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: read oid 429 snap -1 2014-07-01 17:02:43,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:1961: snap_create 2014-07-01 17:02:43,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: expect (ObjNum 428 snap 0 seq_num 428) 2014-07-01 17:02:44,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: finishing write tid 1 to mira0437250-330 2014-07-01 17:02:44,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: finishing write tid 2 to mira0437250-330 2014-07-01 17:02:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: finishing write tid 3 to mira0437250-330 2014-07-01 17:02:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: finishing write tid 5 to mira0437250-330 2014-07-01 17:02:44,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: finishing write tid 6 to mira0437250-330 2014-07-01 17:02:44,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 550 (ObjNum 1035 snap 204 seq_num 1035) dirty exists 2014-07-01 17:02:44,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 1 to mira0435601-144 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 2 to mira0435601-144 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 3 to mira0435601-144 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 5 to mira0435601-144 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 6 to mira0435601-144 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 144 v 131 (ObjNum 1959 snap 0 seq_num 1959) dirty exists 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: done (11 left) 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: done (10 left) 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: done (9 left) 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: done (8 left) 2014-07-01 17:02:44,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3143: done (7 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: done (6 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3145: done (5 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: done (4 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3147: done (3 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: done (2 left) 2014-07-01 17:02:44,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 90 current snap is 0 2014-07-01 17:02:44,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: seq_num 1960 ranges {2162688=405504} 2014-07-01 17:02:44,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: writing mira0435601-90 from 2162688 to 2568192 tid 1 2014-07-01 17:02:44,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:3152: delete oid 268 current snap is 0 2014-07-01 17:02:44,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3149: expect (ObjNum 1327 snap 0 seq_num 1327) 2014-07-01 17:02:44,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1959: done (2 left) 2014-07-01 17:02:44,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1960: done (1 left) 2014-07-01 17:02:44,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1961: done (0 left) 2014-07-01 17:02:44,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: rollback oid 260 current snap is 205 2014-07-01 17:02:44,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 260 to 175 2014-07-01 17:02:44,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: read oid 256 snap 156 2014-07-01 17:02:44,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: read oid 428 snap 149 2014-07-01 17:02:44,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: read oid 436 snap -1 2014-07-01 17:02:44,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: delete oid 324 current snap is 205 2014-07-01 17:02:44,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1966: done (4 left) 2014-07-01 17:02:44,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: read oid 368 snap -1 2014-07-01 17:02:44,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:1968: rollback oid 387 current snap is 205 2014-07-01 17:02:44,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 387 to 175 2014-07-01 17:02:44,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1969: snap_remove snap 200 2014-07-01 17:02:44,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 0 (ObjNum 49 snap 0 seq_num 3) dirty dne 2014-07-01 17:02:44,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 606 (ObjNum 259 snap 0 seq_num 259) dirty exists 2014-07-01 17:02:44,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: expect (ObjNum 367 snap 0 seq_num 367) 2014-07-01 17:02:45,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:3150: expect (ObjNum 1580 snap 0 seq_num 1580) 2014-07-01 17:02:46,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: expect (ObjNum 435 snap 0 seq_num 435) 2014-07-01 17:02:46,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: expect (ObjNum 255 snap 0 seq_num 255) 2014-07-01 17:02:46,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1962: done (6 left) 2014-07-01 17:02:46,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1963: done (5 left) 2014-07-01 17:02:46,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:1964: done (4 left) 2014-07-01 17:02:46,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1965: done (3 left) 2014-07-01 17:02:46,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:1967: done (2 left) 2014-07-01 17:02:46,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1968: done (1 left) 2014-07-01 17:02:46,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1969: done (0 left) 2014-07-01 17:02:46,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: delete oid 412 current snap is 205 2014-07-01 17:02:46,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:1970: done (0 left) 2014-07-01 17:02:46,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: snap_remove snap 174 2014-07-01 17:02:47,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:1971: done (0 left) 2014-07-01 17:02:47,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: read oid 276 snap -1 2014-07-01 17:02:47,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: rollback oid 468 current snap is 205 2014-07-01 17:02:47,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 468 to 189 2014-07-01 17:02:47,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: read oid 384 snap -1 2014-07-01 17:02:47,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: write oid 15 current snap is 205 2014-07-01 17:02:47,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: seq_num 1038 ranges {574934=1} 2014-07-01 17:02:47,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: writing mira0437250-15 from 574934 to 574935 tid 1 2014-07-01 17:02:47,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1976: delete oid 302 current snap is 205 2014-07-01 17:02:47,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: expect (ObjNum 383 snap 0 seq_num 383) 2014-07-01 17:02:48,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:1974: done (4 left) 2014-07-01 17:02:48,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1976: done (3 left) 2014-07-01 17:02:48,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: write oid 319 current snap is 205 2014-07-01 17:02:48,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: seq_num 1040 ranges {766418=593476,2045005=563616,3127975=421116,3966417=1} 2014-07-01 17:02:48,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: writing mira0437250-319 from 766418 to 1359894 tid 1 2014-07-01 17:02:48,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: writing mira0437250-319 from 2045005 to 2608621 tid 2 2014-07-01 17:02:48,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: writing mira0437250-319 from 3127975 to 3549091 tid 3 2014-07-01 17:02:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: writing mira0437250-319 from 3966417 to 3966418 tid 4 2014-07-01 17:02:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1978: snap_remove snap 201 2014-07-01 17:02:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 1 to mira0437250-15 2014-07-01 17:02:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 3 to mira0437250-15 2014-07-01 17:02:48,969.969 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: finishing write tid 4 to mira0437250-15 2014-07-01 17:02:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 807 (ObjNum 1038 snap 205 seq_num 1038) dirty exists 2014-07-01 17:02:48,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: expect (ObjNum 275 snap 0 seq_num 275) 2014-07-01 17:02:49,593.593 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 1, 2, 0] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:49,594.594 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:49,595.595 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [3, 5, 4, 1, 2, 0] 2014-07-01 17:02:49,595.595 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 17:02:49,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: finishing write tid 1 to mira0435601-90 2014-07-01 17:02:49,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: finishing write tid 3 to mira0435601-90 2014-07-01 17:02:49,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: finishing write tid 4 to mira0435601-90 2014-07-01 17:02:49,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 90 v 126 (ObjNum 1960 snap 0 seq_num 1960) dirty exists 2014-07-01 17:02:49,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:3149: done (3 left) 2014-07-01 17:02:49,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:3150: done (2 left) 2014-07-01 17:02:49,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: done (1 left) 2014-07-01 17:02:49,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:3152: done (0 left) 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: read oid 188 snap -1 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 76 current snap is 0 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: seq_num 1962 ranges {3121152=598016,3719168=401408} 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: writing mira0435601-76 from 3121152 to 3719168 tid 1 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: writing mira0435601-76 from 3719168 to 4120576 tid 2 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3155: delete oid 470 current snap is 0 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: expect (ObjNum 1741 snap 0 seq_num 1741) 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 468 v 573 (ObjNum 467 snap 0 seq_num 467) dirty exists 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 1 to mira0437250-319 2014-07-01 17:02:49,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 2 to mira0437250-319 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1972: done (4 left) 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1973: done (3 left) 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1975: done (2 left) 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1978: done (1 left) 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: read oid 345 snap -1 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 3 to mira0437250-319 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 4 to mira0437250-319 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 6 to mira0437250-319 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: finishing write tid 7 to mira0437250-319 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 1077 (ObjNum 1040 snap 205 seq_num 1040) dirty exists 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1977: done (1 left) 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: write oid 429 current snap is 205 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: seq_num 1041 ranges {558800=600352,1606689=152111} 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: writing mira0437250-429 from 558800 to 1159152 tid 1 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: writing mira0437250-429 from 1606689 to 1758800 tid 2 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: rollback oid 358 current snap is 205 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 358 to 189 2014-07-01 17:02:49,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: expect (ObjNum 805 snap 130 seq_num 805) 2014-07-01 17:02:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 358 v 752 (ObjNum 357 snap 0 seq_num 357) dirty exists 2014-07-01 17:02:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 1 to mira0437250-429 2014-07-01 17:02:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 2 to mira0437250-429 2014-07-01 17:02:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 4 to mira0437250-429 2014-07-01 17:02:50,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:1979: done (2 left) 2014-07-01 17:02:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1981: done (1 left) 2014-07-01 17:02:50,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: snap_create 2014-07-01 17:02:50,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: finishing write tid 5 to mira0437250-429 2014-07-01 17:02:50,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 823 (ObjNum 1041 snap 205 seq_num 1041) dirty exists 2014-07-01 17:02:50,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1980: done (1 left) 2014-07-01 17:02:50,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1982: done (0 left) 2014-07-01 17:02:50,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:1983: snap_remove snap 187 2014-07-01 17:02:50,713.713 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 17:02:51,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:1983: done (0 left) 2014-07-01 17:02:51,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1984: rollback oid 481 current snap is 206 2014-07-01 17:02:51,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 481 to 191 2014-07-01 17:02:51,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: read oid 304 snap -1 2014-07-01 17:02:51,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: read oid 441 snap 189 2014-07-01 17:02:51,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: write oid 402 current snap is 206 2014-07-01 17:02:51,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: seq_num 1042 ranges {771406=572157,1971405=1} 2014-07-01 17:02:51,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: writing mira0437250-402 from 771406 to 1343563 tid 1 2014-07-01 17:02:51,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: writing mira0437250-402 from 1971405 to 1971406 tid 2 2014-07-01 17:02:51,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: write oid 353 current snap is 206 2014-07-01 17:02:51,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: seq_num 1043 ranges {467659=543961,1515209=152450} 2014-07-01 17:02:51,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: writing mira0437250-353 from 467659 to 1011620 tid 1 2014-07-01 17:02:51,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: writing mira0437250-353 from 1515209 to 1667659 tid 2 2014-07-01 17:02:51,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1989: delete oid 475 current snap is 206 2014-07-01 17:02:51,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: expect (ObjNum 575 snap 28 seq_num 575) 2014-07-01 17:02:52,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: expect (ObjNum 303 snap 0 seq_num 303) 2014-07-01 17:02:52,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: finishing write tid 1 to mira0437250-402 2014-07-01 17:02:52,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: finishing write tid 2 to mira0437250-402 2014-07-01 17:02:52,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: finishing write tid 4 to mira0437250-402 2014-07-01 17:02:52,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: finishing write tid 5 to mira0437250-402 2014-07-01 17:02:52,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 774 (ObjNum 1042 snap 206 seq_num 1042) dirty exists 2014-07-01 17:02:52,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 1 to mira0437250-353 2014-07-01 17:02:52,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 2 to mira0437250-353 2014-07-01 17:02:52,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 4 to mira0437250-353 2014-07-01 17:02:52,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:1985: done (5 left) 2014-07-01 17:02:52,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1986: done (4 left) 2014-07-01 17:02:52,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1987: done (3 left) 2014-07-01 17:02:52,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1989: done (2 left) 2014-07-01 17:02:52,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: write oid 472 current snap is 206 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: seq_num 1045 ranges {755271=523666,1723251=549664,3011031=144240} 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: writing mira0437250-472 from 755271 to 1278937 tid 1 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: writing mira0437250-472 from 1723251 to 2272915 tid 2 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: writing mira0437250-472 from 3011031 to 3155271 tid 3 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1991: snap_remove snap 204 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: finishing write tid 5 to mira0437250-353 2014-07-01 17:02:52,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 353 v 600 (ObjNum 1043 snap 206 seq_num 1043) dirty exists 2014-07-01 17:02:52,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:1988: done (3 left) 2014-07-01 17:02:52,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1991: done (2 left) 2014-07-01 17:02:52,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: write oid 114 current snap is 206 2014-07-01 17:02:52,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: seq_num 1046 ranges {566853=495470,1669063=771283,3080033=286820} 2014-07-01 17:02:52,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: writing mira0437250-114 from 566853 to 1062323 tid 1 2014-07-01 17:02:52,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: writing mira0437250-114 from 1669063 to 2440346 tid 2 2014-07-01 17:02:52,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: writing mira0437250-114 from 3080033 to 3366853 tid 3 2014-07-01 17:02:52,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: write oid 351 current snap is 206 2014-07-01 17:02:52,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: seq_num 1047 ranges {111681=1} 2014-07-01 17:02:52,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: writing mira0437250-351 from 111681 to 111682 tid 1 2014-07-01 17:02:52,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:1994: snap_create 2014-07-01 17:02:52,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: finishing write tid 1 to mira0435601-76 2014-07-01 17:02:52,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: finishing write tid 2 to mira0435601-76 2014-07-01 17:02:52,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: finishing write tid 4 to mira0435601-76 2014-07-01 17:02:52,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: finishing write tid 5 to mira0435601-76 2014-07-01 17:02:52,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 287 (ObjNum 1962 snap 0 seq_num 1962) dirty exists 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: done (2 left) 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: done (1 left) 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3155: done (0 left) 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3156: read oid 184 snap -1 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: read oid 364 snap -1 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 178 current snap is 0 2014-07-01 17:02:52,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: seq_num 1964 ranges {2052096=499712,2551808=438272,2990080=634880,3624960=532480} 2014-07-01 17:02:52,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: writing mira0435601-178 from 2052096 to 2551808 tid 1 2014-07-01 17:02:52,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: writing mira0435601-178 from 2551808 to 2990080 tid 2 2014-07-01 17:02:52,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: writing mira0435601-178 from 2990080 to 3624960 tid 3 2014-07-01 17:02:52,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: writing mira0435601-178 from 3624960 to 4157440 tid 4 2014-07-01 17:02:52,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3159: read oid 277 snap -1 2014-07-01 17:02:52,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: read oid 284 snap -1 2014-07-01 17:02:52,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: read oid 146 snap -1 2014-07-01 17:02:52,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 99 current snap is 0 2014-07-01 17:02:52,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: seq_num 1965 ranges {2473984=770048,3244032=618496,3862528=458752,4321280=126976} 2014-07-01 17:02:52,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: writing mira0435601-99 from 2473984 to 3244032 tid 1 2014-07-01 17:02:53,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: writing mira0435601-99 from 3244032 to 3862528 tid 2 2014-07-01 17:02:53,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: writing mira0435601-99 from 3862528 to 4321280 tid 3 2014-07-01 17:02:53,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: writing mira0435601-99 from 4321280 to 4448256 tid 4 2014-07-01 17:02:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: read oid 344 snap -1 2014-07-01 17:02:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: read oid 417 snap -1 2014-07-01 17:02:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3165: read oid 239 snap -1 2014-07-01 17:02:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 17:02:53,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: seq_num 1966 ranges {7057408=446464,7503872=602112,8105984=602112} 2014-07-01 17:02:53,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: writing mira0435601-394 from 7057408 to 7503872 tid 1 2014-07-01 17:02:53,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: writing mira0435601-394 from 7503872 to 8105984 tid 2 2014-07-01 17:02:53,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: writing mira0435601-394 from 8105984 to 8708096 tid 3 2014-07-01 17:02:53,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 414 current snap is 0 2014-07-01 17:02:53,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: seq_num 1967 ranges {2220032=720896,2940928=434176,3375104=368640} 2014-07-01 17:02:53,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: finishing write tid 1 to mira0437250-351 2014-07-01 17:02:53,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: finishing write tid 3 to mira0437250-351 2014-07-01 17:02:53,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: writing mira0435601-414 from 2220032 to 2940928 tid 1 2014-07-01 17:02:53,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: writing mira0435601-414 from 2940928 to 3375104 tid 2 2014-07-01 17:02:53,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: writing mira0435601-414 from 3375104 to 3743744 tid 3 2014-07-01 17:02:53,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3168: read oid 38 snap -1 2014-07-01 17:02:53,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 287 current snap is 0 2014-07-01 17:02:53,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: seq_num 1968 ranges {2150400=589824,2740224=417792,3158016=385024} 2014-07-01 17:02:53,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: writing mira0435601-287 from 2150400 to 2740224 tid 1 2014-07-01 17:02:53,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: writing mira0435601-287 from 2740224 to 3158016 tid 2 2014-07-01 17:02:53,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: writing mira0435601-287 from 3158016 to 3543040 tid 3 2014-07-01 17:02:53,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 42 current snap is 0 2014-07-01 17:02:53,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: seq_num 1969 ranges {2940928=217088} 2014-07-01 17:02:53,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: writing mira0435601-42 from 2940928 to 3158016 tid 1 2014-07-01 17:02:53,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: read oid 28 snap -1 2014-07-01 17:02:53,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 17:02:53,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:1994: done (4 left) 2014-07-01 17:02:53,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: read oid 397 snap -1 2014-07-01 17:02:53,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:1996: snap_remove snap 198 2014-07-01 17:02:54,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 1 to mira0437250-472 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 2 to mira0437250-472 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 3 to mira0437250-472 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: finishing write tid 4 to mira0437250-351 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 479 (ObjNum 1047 snap 206 seq_num 1047) dirty exists 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 5 to mira0437250-472 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: finishing write tid 6 to mira0437250-472 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 472 v 590 (ObjNum 1045 snap 206 seq_num 1045) dirty exists 2014-07-01 17:02:54,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: expect (ObjNum 1448 snap 0 seq_num 1448) 2014-07-01 17:02:54,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 837 (ObjNum 693 snap 88 seq_num 693) dirty exists 2014-07-01 17:02:54,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 1 to mira0437250-114 2014-07-01 17:02:54,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 2 to mira0437250-114 2014-07-01 17:02:54,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 3 to mira0437250-114 2014-07-01 17:02:54,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 5 to mira0437250-114 2014-07-01 17:02:54,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: finishing write tid 6 to mira0437250-114 2014-07-01 17:02:54,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 1111 (ObjNum 1046 snap 206 seq_num 1046) dirty exists 2014-07-01 17:02:54,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: expect (ObjNum 396 snap 0 seq_num 396) 2014-07-01 17:02:54,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3156: expect (ObjNum 183 snap 0 seq_num 183) 2014-07-01 17:02:54,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1984: done (5 left) 2014-07-01 17:02:54,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:1990: done (4 left) 2014-07-01 17:02:54,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:1992: done (3 left) 2014-07-01 17:02:54,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:1993: done (2 left) 2014-07-01 17:02:54,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:1995: done (1 left) 2014-07-01 17:02:54,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1996: done (0 left) 2014-07-01 17:02:54,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: write oid 318 current snap is 207 2014-07-01 17:02:54,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: seq_num 1048 ranges {456512=400000} 2014-07-01 17:02:55,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: writing mira0437250-318 from 456512 to 856512 tid 1 2014-07-01 17:02:55,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: snap_remove snap 149 2014-07-01 17:02:55,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3159: expect (ObjNum 909 snap 0 seq_num 909) 2014-07-01 17:02:55,728.728 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0] out_osds: [1] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:02:55,728.728 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:02:55,729.729 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 17:02:55,729.729 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 17:02:56,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:1998: done (1 left) 2014-07-01 17:02:56,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:1999: rollback oid 482 current snap is 207 2014-07-01 17:02:56,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 482 to 191 2014-07-01 17:02:56,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: rollback oid 475 current snap is 207 2014-07-01 17:02:56,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 475 to 179 2014-07-01 17:02:56,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: rollback oid 235 current snap is 207 2014-07-01 17:02:56,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 235 to 202 2014-07-01 17:02:56,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: write oid 39 current snap is 207 2014-07-01 17:02:56,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: seq_num 1049 ranges {551741=1} 2014-07-01 17:02:56,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: writing mira0437250-39 from 551741 to 551742 tid 1 2014-07-01 17:02:56,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: delete oid 493 current snap is 207 2014-07-01 17:02:56,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 1 to mira0435601-178 2014-07-01 17:02:56,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 2 to mira0435601-178 2014-07-01 17:02:56,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 3 to mira0435601-178 2014-07-01 17:02:56,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 4 to mira0435601-178 2014-07-01 17:02:56,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 6 to mira0435601-178 2014-07-01 17:02:56,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: finishing write tid 1 to mira0435601-287 2014-07-01 17:02:56,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: finishing write tid 2 to mira0435601-287 2014-07-01 17:02:56,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: expect (ObjNum 1877 snap 0 seq_num 1877) 2014-07-01 17:02:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 0 (ObjNum 16742208 snap 0 seq_num 17382480) dirty dne 2014-07-01 17:02:56,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: expect (ObjNum 1928 snap 0 seq_num 1928) 2014-07-01 17:02:56,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: expect (ObjNum 1489 snap 0 seq_num 1489) 2014-07-01 17:02:57,059.059 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 17:02:57,066.066 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 17:02:57,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 991 (ObjNum 971 snap 179 seq_num 971) dirty exists 2014-07-01 17:02:57,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 993 (ObjNum 474 snap 0 seq_num 474) dirty exists 2014-07-01 17:02:58,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:1999: done (5 left) 2014-07-01 17:02:58,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2000: done (4 left) 2014-07-01 17:02:58,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2001: done (3 left) 2014-07-01 17:02:58,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2003: done (2 left) 2014-07-01 17:02:58,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:2004: snap_create 2014-07-01 17:02:59,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: expect (ObjNum 1672 snap 0 seq_num 1672) 2014-07-01 17:02:59,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2004: done (2 left) 2014-07-01 17:02:59,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2005: snap_create 2014-07-01 17:02:59,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: finishing write tid 1 to mira0437250-318 2014-07-01 17:03:00,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:2005: done (2 left) 2014-07-01 17:03:00,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: snap_remove snap 202 2014-07-01 17:03:00,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: finishing write tid 3 to mira0435601-287 2014-07-01 17:03:00,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: finishing write tid 5 to mira0435601-287 2014-07-01 17:03:00,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: finishing write tid 7 to mira0435601-178 2014-07-01 17:03:00,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 352 (ObjNum 1964 snap 0 seq_num 1964) dirty exists 2014-07-01 17:03:00,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: finishing write tid 6 to mira0435601-287 2014-07-01 17:03:00,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 287 v 196 (ObjNum 1968 snap 0 seq_num 1968) dirty exists 2014-07-01 17:03:00,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 1 to mira0435601-99 2014-07-01 17:03:00,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3156: done (15 left) 2014-07-01 17:03:00,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: done (14 left) 2014-07-01 17:03:00,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: done (13 left) 2014-07-01 17:03:00,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3159: done (12 left) 2014-07-01 17:03:00,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: done (11 left) 2014-07-01 17:03:00,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: done (10 left) 2014-07-01 17:03:01,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: done (9 left) 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: done (8 left) 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: done (7 left) 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3172: read oid 446 snap -1 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 2 to mira0435601-99 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3165: expect (ObjNum 1620 snap 0 seq_num 1620) 2014-07-01 17:03:01,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3168: expect (ObjNum 1881 snap 0 seq_num 1881) 2014-07-01 17:03:01,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2006: done (2 left) 2014-07-01 17:03:01,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: rollback oid 269 current snap is 209 2014-07-01 17:03:01,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 269 to 192 2014-07-01 17:03:01,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: read oid 476 snap -1 2014-07-01 17:03:01,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:2009: snap_remove snap 175 2014-07-01 17:03:01,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: expect (ObjNum 957 snap 175 seq_num 957) 2014-07-01 17:03:01,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 269 v 718 (ObjNum 268 snap 0 seq_num 268) dirty exists 2014-07-01 17:03:02,064.064 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:02,065.065 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:02,065.065 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 17:03:02,065.065 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:03:02,065.065 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:03:02,066.066 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 17:03:02,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2007: done (4 left) 2014-07-01 17:03:02,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2008: done (3 left) 2014-07-01 17:03:02,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2009: done (2 left) 2014-07-01 17:03:02,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:2010: snap_remove snap 191 2014-07-01 17:03:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: finishing write tid 1 to mira0435601-394 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: finishing write tid 2 to mira0435601-394 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: finishing write tid 3 to mira0435601-394 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: finishing write tid 5 to mira0435601-394 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 3 to mira0435601-99 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 4 to mira0435601-99 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 6 to mira0435601-99 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: finishing write tid 1 to mira0435601-414 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: finishing write tid 2 to mira0435601-414 2014-07-01 17:03:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: finishing write tid 3 to mira0435601-414 2014-07-01 17:03:02,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: finishing write tid 1 to mira0435601-42 2014-07-01 17:03:02,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: finishing write tid 5 to mira0435601-414 2014-07-01 17:03:02,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: finishing write tid 3 to mira0435601-42 2014-07-01 17:03:02,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: expect (ObjNum 1893 snap 0 seq_num 1893) 2014-07-01 17:03:02,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: finishing write tid 3 to mira0437250-318 2014-07-01 17:03:02,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: finishing write tid 4 to mira0437250-318 2014-07-01 17:03:02,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 484 (ObjNum 1048 snap 207 seq_num 1048) dirty exists 2014-07-01 17:03:03,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:1997: done (2 left) 2014-07-01 17:03:03,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2010: done (1 left) 2014-07-01 17:03:03,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: read oid 15 snap -1 2014-07-01 17:03:03,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: read oid 190 snap -1 2014-07-01 17:03:03,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: write oid 488 current snap is 209 2014-07-01 17:03:03,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: seq_num 1051 ranges {557753=695484,2019179=538574} 2014-07-01 17:03:03,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: writing mira0437250-488 from 557753 to 1253237 tid 1 2014-07-01 17:03:03,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: writing mira0437250-488 from 2019179 to 2557753 tid 2 2014-07-01 17:03:03,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2014: delete oid 282 current snap is 209 2014-07-01 17:03:03,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: expect (ObjNum 1038 snap 205 seq_num 1038) 2014-07-01 17:03:05,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: finishing write tid 1 to mira0437250-39 2014-07-01 17:03:05,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: finishing write tid 3 to mira0437250-39 2014-07-01 17:03:06,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: finishing write tid 6 to mira0435601-394 2014-07-01 17:03:06,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 198 (ObjNum 1966 snap 0 seq_num 1966) dirty exists 2014-07-01 17:03:06,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: finishing write tid 6 to mira0435601-414 2014-07-01 17:03:06,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 414 v 371 (ObjNum 1967 snap 0 seq_num 1967) dirty exists 2014-07-01 17:03:06,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: finishing write tid 4 to mira0435601-42 2014-07-01 17:03:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 42 v 235 (ObjNum 1969 snap 0 seq_num 1969) dirty exists 2014-07-01 17:03:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: finishing write tid 7 to mira0435601-99 2014-07-01 17:03:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 99 v 160 (ObjNum 1965 snap 0 seq_num 1965) dirty exists 2014-07-01 17:03:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: done (7 left) 2014-07-01 17:03:06,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: done (6 left) 2014-07-01 17:03:06,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:3165: done (5 left) 2014-07-01 17:03:06,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: done (4 left) 2014-07-01 17:03:06,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: done (3 left) 2014-07-01 17:03:06,292.292 INFO:teuthology.task.rados.rados.0.mira043.stdout:3168: done (2 left) 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: done (1 left) 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3172: done (0 left) 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 462 current snap is 0 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: seq_num 1970 ranges {2310144=540672,2850816=401408} 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: writing mira0435601-462 from 2310144 to 2850816 tid 1 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: writing mira0435601-462 from 2850816 to 3252224 tid 2 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3174: read oid 182 snap -1 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 128 current snap is 0 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: seq_num 1971 ranges {0=409600,409600=401408} 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: writing mira0435601-128 from 0 to 409600 tid 1 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: writing mira0435601-128 from 409600 to 811008 tid 2 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 196 current snap is 0 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: seq_num 1972 ranges {1773568=438272,2211840=798720,3010560=684032,3694592=122880} 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: writing mira0435601-196 from 1773568 to 2211840 tid 1 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: writing mira0435601-196 from 2211840 to 3010560 tid 2 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: writing mira0435601-196 from 3010560 to 3694592 tid 3 2014-07-01 17:03:06,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: writing mira0435601-196 from 3694592 to 3817472 tid 4 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3177: read oid 100 snap -1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3174: done (4 left) 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 271 current snap is 0 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: seq_num 1973 ranges {176128=712704,888832=425984,1314816=778240} 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: writing mira0435601-271 from 176128 to 888832 tid 1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: writing mira0435601-271 from 888832 to 1314816 tid 2 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: writing mira0435601-271 from 1314816 to 2093056 tid 3 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: seq_num 1974 ranges {2691072=737280} 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: writing mira0435601-228 from 2691072 to 3428352 tid 1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3180: read oid 457 snap -1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3177: done (6 left) 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3181: read oid 342 snap -1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 12 current snap is 0 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: seq_num 1975 ranges {4087808=659456,4747264=638976,5386240=163840} 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: writing mira0435601-12 from 4087808 to 4747264 tid 1 2014-07-01 17:03:06,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: writing mira0435601-12 from 4747264 to 5386240 tid 2 2014-07-01 17:03:06,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: writing mira0435601-12 from 5386240 to 5550080 tid 3 2014-07-01 17:03:06,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3183: delete oid 450 current snap is 0 2014-07-01 17:03:06,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3180: expect (ObjNum 1911 snap 0 seq_num 1911) 2014-07-01 17:03:07,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: finishing write tid 1 to mira0437250-488 2014-07-01 17:03:07,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: finishing write tid 2 to mira0437250-488 2014-07-01 17:03:07,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: finishing write tid 4 to mira0437250-488 2014-07-01 17:03:07,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: finishing write tid 5 to mira0437250-488 2014-07-01 17:03:07,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 618 (ObjNum 1051 snap 209 seq_num 1051) dirty exists 2014-07-01 17:03:07,542.542 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:07,542.542 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:07,543.543 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 17:03:07,543.543 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:03:07,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: finishing write tid 4 to mira0437250-39 2014-07-01 17:03:07,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 39 v 873 (ObjNum 1049 snap 207 seq_num 1049) dirty exists 2014-07-01 17:03:08,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2002: done (4 left) 2014-07-01 17:03:08,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2011: done (3 left) 2014-07-01 17:03:08,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2012: done (2 left) 2014-07-01 17:03:08,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2013: done (1 left) 2014-07-01 17:03:08,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2014: done (0 left) 2014-07-01 17:03:08,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: read oid 27 snap -1 2014-07-01 17:03:08,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: write oid 330 current snap is 209 2014-07-01 17:03:08,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: seq_num 1053 ranges {596789=674164,1845395=764473,2996788=1} 2014-07-01 17:03:08,246.246 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:03:08,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: writing mira0437250-330 from 596789 to 1270953 tid 1 2014-07-01 17:03:08,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: writing mira0437250-330 from 1845395 to 2609868 tid 2 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: writing mira0437250-330 from 2996788 to 2996789 tid 3 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: snap_create 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: expect (ObjNum 647 snap 67 seq_num 647) 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2015: done (2 left) 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2017: done (1 left) 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: write oid 471 current snap is 210 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: seq_num 1054 ranges {540595=495569,1505256=469741,2768812=545751,3740594=1} 2014-07-01 17:03:08,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0437250-471 from 540595 to 1036164 tid 1 2014-07-01 17:03:08,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0437250-471 from 1505256 to 1974997 tid 2 2014-07-01 17:03:08,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0437250-471 from 2768812 to 3314563 tid 3 2014-07-01 17:03:08,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: writing mira0437250-471 from 3740594 to 3740595 tid 4 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: write oid 122 current snap is 210 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: seq_num 1055 ranges {601776=400000} 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: writing mira0437250-122 from 601776 to 1001776 tid 1 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2020: snap_create 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 1 to mira0437250-471 2014-07-01 17:03:08,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 2 to mira0437250-471 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: finishing write tid 1 to mira0437250-330 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 3 to mira0437250-471 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 4 to mira0437250-471 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 6 to mira0437250-471 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: finishing write tid 7 to mira0437250-471 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 739 (ObjNum 1054 snap 210 seq_num 1054) dirty exists 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: finishing write tid 2 to mira0437250-330 2014-07-01 17:03:08,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: finishing write tid 3 to mira0437250-330 2014-07-01 17:03:08,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: finishing write tid 5 to mira0437250-330 2014-07-01 17:03:08,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: finishing write tid 6 to mira0437250-330 2014-07-01 17:03:08,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 583 (ObjNum 1053 snap 209 seq_num 1053) dirty exists 2014-07-01 17:03:09,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: finishing write tid 1 to mira0435601-228 2014-07-01 17:03:09,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: finishing write tid 3 to mira0435601-228 2014-07-01 17:03:09,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: finishing write tid 1 to mira0435601-271 2014-07-01 17:03:09,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: finishing write tid 2 to mira0435601-271 2014-07-01 17:03:09,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: finishing write tid 3 to mira0435601-271 2014-07-01 17:03:09,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: finishing write tid 1 to mira0435601-12 2014-07-01 17:03:09,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: finishing write tid 5 to mira0435601-271 2014-07-01 17:03:09,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: finishing write tid 2 to mira0435601-12 2014-07-01 17:03:09,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: finishing write tid 3 to mira0435601-12 2014-07-01 17:03:09,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: finishing write tid 5 to mira0435601-12 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: finishing write tid 4 to mira0435601-228 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 164 (ObjNum 1974 snap 0 seq_num 1974) dirty exists 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: finishing write tid 6 to mira0435601-271 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 271 v 227 (ObjNum 1973 snap 0 seq_num 1973) dirty exists 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: finishing write tid 6 to mira0435601-12 2014-07-01 17:03:09,135.135 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 12 v 125 (ObjNum 1975 snap 0 seq_num 1975) dirty exists 2014-07-01 17:03:09,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 1 to mira0435601-196 2014-07-01 17:03:09,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: done (8 left) 2014-07-01 17:03:09,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: done (7 left) 2014-07-01 17:03:09,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3180: done (6 left) 2014-07-01 17:03:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3181: done (5 left) 2014-07-01 17:03:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: done (4 left) 2014-07-01 17:03:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3183: done (3 left) 2014-07-01 17:03:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3184: read oid 91 snap -1 2014-07-01 17:03:09,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 2 to mira0435601-196 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 3 to mira0435601-196 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 4 to mira0435601-196 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 6 to mira0435601-196 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: finishing write tid 7 to mira0435601-196 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 357 (ObjNum 1972 snap 0 seq_num 1972) dirty exists 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 1 to mira0435601-128 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 2 to mira0435601-128 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 4 to mira0435601-128 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 5 to mira0435601-128 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 216 (ObjNum 1971 snap 0 seq_num 1971) dirty exists 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: finishing write tid 1 to mira0435601-462 2014-07-01 17:03:09,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: finishing write tid 2 to mira0435601-462 2014-07-01 17:03:09,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: finishing write tid 4 to mira0435601-462 2014-07-01 17:03:09,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: finishing write tid 5 to mira0435601-462 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 201 (ObjNum 1970 snap 0 seq_num 1970) dirty exists 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: done (3 left) 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: done (2 left) 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: done (1 left) 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 17:03:09,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: seq_num 1977 ranges {4308992=155648} 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: writing mira0435601-397 from 4308992 to 4464640 tid 1 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 30 current snap is 0 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: seq_num 1978 ranges {4263936=28672} 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: writing mira0435601-30 from 4263936 to 4292608 tid 1 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 481 current snap is 0 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: seq_num 1979 ranges {3604480=749568} 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: writing mira0435601-481 from 3604480 to 4354048 tid 1 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3188: read oid 66 snap -1 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: finishing write tid 1 to mira0435601-397 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: finishing write tid 3 to mira0435601-397 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: finishing write tid 4 to mira0435601-397 2014-07-01 17:03:09,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 162 (ObjNum 1977 snap 0 seq_num 1977) dirty exists 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 1 to mira0435601-30 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 3 to mira0435601-30 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 4 to mira0435601-30 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 166 (ObjNum 1978 snap 0 seq_num 1978) dirty exists 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3184: done (4 left) 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: done (3 left) 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: done (2 left) 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3189: read oid 466 snap -1 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 158 current snap is 0 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: seq_num 1980 ranges {1679360=778240,2457600=606208,3063808=573440,3637248=24576} 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0435601-158 from 1679360 to 2457600 tid 1 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0435601-158 from 2457600 to 3063808 tid 2 2014-07-01 17:03:09,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0435601-158 from 3063808 to 3637248 tid 3 2014-07-01 17:03:09,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0435601-158 from 3637248 to 3661824 tid 4 2014-07-01 17:03:09,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 59 current snap is 0 2014-07-01 17:03:09,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: seq_num 1981 ranges {0=651264,651264=786432,1437696=417792} 2014-07-01 17:03:09,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: writing mira0435601-59 from 0 to 651264 tid 1 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: writing mira0435601-59 from 651264 to 1437696 tid 2 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: writing mira0435601-59 from 1437696 to 1855488 tid 3 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 244 current snap is 0 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: seq_num 1982 ranges {3690496=520192,4210688=770048,4980736=434176} 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: writing mira0435601-244 from 3690496 to 4210688 tid 1 2014-07-01 17:03:09,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: writing mira0435601-244 from 4210688 to 4980736 tid 2 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: writing mira0435601-244 from 4980736 to 5414912 tid 3 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:3193: delete oid 224 current snap is 0 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:3188: expect (ObjNum 646 snap 0 seq_num 646) 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2016: done (3 left) 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2018: done (2 left) 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2020: done (1 left) 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: read oid 373 snap 158 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: read oid 491 snap -1 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: snap_remove snap 179 2014-07-01 17:03:09,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: expect (ObjNum 372 snap 0 seq_num 372) 2014-07-01 17:03:11,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 1 to mira0435601-158 2014-07-01 17:03:11,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 2 to mira0435601-158 2014-07-01 17:03:11,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 3 to mira0435601-158 2014-07-01 17:03:11,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 4 to mira0435601-158 2014-07-01 17:03:11,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 6 to mira0435601-158 2014-07-01 17:03:11,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 7 to mira0435601-158 2014-07-01 17:03:11,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 158 v 204 (ObjNum 1980 snap 0 seq_num 1980) dirty exists 2014-07-01 17:03:11,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3188: done (6 left) 2014-07-01 17:03:11,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3189: done (5 left) 2014-07-01 17:03:11,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: done (4 left) 2014-07-01 17:03:11,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3193: done (3 left) 2014-07-01 17:03:11,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: seq_num 1984 ranges {1617920=225280} 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: writing mira0435601-223 from 1617920 to 1843200 tid 1 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3195: read oid 329 snap -1 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: seq_num 1985 ranges {0=741376,741376=401408} 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: writing mira0435601-323 from 0 to 741376 tid 1 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: writing mira0435601-323 from 741376 to 1142784 tid 2 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 370 current snap is 0 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: seq_num 1986 ranges {2572288=770048,3342336=598016,3940352=569344,4509696=438272} 2014-07-01 17:03:11,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: writing mira0435601-370 from 2572288 to 3342336 tid 1 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: writing mira0435601-370 from 3342336 to 3940352 tid 2 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: writing mira0435601-370 from 3940352 to 4509696 tid 3 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: writing mira0435601-370 from 4509696 to 4947968 tid 4 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 451 current snap is 0 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: seq_num 1987 ranges {1433600=643072,2076672=425984,2502656=798720,3301376=380928} 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0435601-451 from 1433600 to 2076672 tid 1 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0435601-451 from 2076672 to 2502656 tid 2 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0435601-451 from 2502656 to 3301376 tid 3 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0435601-451 from 3301376 to 3682304 tid 4 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: read oid 447 snap -1 2014-07-01 17:03:11,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3195: expect (ObjNum 1432 snap 0 seq_num 1432) 2014-07-01 17:03:12,167.167 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 17:03:13,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2021: done (3 left) 2014-07-01 17:03:13,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2023: done (2 left) 2014-07-01 17:03:13,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2024: snap_remove snap 210 2014-07-01 17:03:14,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: expect (ObjNum 490 snap 0 seq_num 490) 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: finishing write tid 1 to mira0435601-323 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: finishing write tid 2 to mira0435601-323 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: finishing write tid 4 to mira0435601-323 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 1 to mira0435601-451 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 2 to mira0435601-451 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 3 to mira0435601-451 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 4 to mira0435601-451 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 6 to mira0435601-451 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: finishing write tid 5 to mira0435601-323 2014-07-01 17:03:14,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 206 (ObjNum 1985 snap 0 seq_num 1985) dirty exists 2014-07-01 17:03:14,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 7 to mira0435601-451 2014-07-01 17:03:14,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 323 (ObjNum 1987 snap 0 seq_num 1987) dirty exists 2014-07-01 17:03:14,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: expect (ObjNum 1584 snap 0 seq_num 1584) 2014-07-01 17:03:14,420.420 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 17:03:14,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2022: done (2 left) 2014-07-01 17:03:14,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2024: done (1 left) 2014-07-01 17:03:14,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2025: delete oid 239 current snap is 211 2014-07-01 17:03:14,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2025: done (1 left) 2014-07-01 17:03:14,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2026: rollback oid 147 current snap is 211 2014-07-01 17:03:14,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 147 to 208 2014-07-01 17:03:14,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: read oid 80 snap -1 2014-07-01 17:03:14,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: read oid 233 snap -1 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: write oid 382 current snap is 211 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: seq_num 1057 ranges {641836=597478,1441835=1} 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: writing mira0437250-382 from 641836 to 1239314 tid 1 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: writing mira0437250-382 from 1441835 to 1441836 tid 2 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: write oid 202 current snap is 211 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: seq_num 1058 ranges {586666=570306,1890616=296050} 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: writing mira0437250-202 from 586666 to 1156972 tid 1 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: writing mira0437250-202 from 1890616 to 2186666 tid 2 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2031: delete oid 384 current snap is 211 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: finishing write tid 1 to mira0437250-382 2014-07-01 17:03:14,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: finishing write tid 2 to mira0437250-382 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: finishing write tid 4 to mira0437250-382 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: finishing write tid 5 to mira0437250-382 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 620 (ObjNum 1057 snap 211 seq_num 1057) dirty exists 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: finishing write tid 1 to mira0437250-202 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: finishing write tid 2 to mira0437250-202 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: finishing write tid 4 to mira0437250-202 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: finishing write tid 5 to mira0437250-202 2014-07-01 17:03:14,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 732 (ObjNum 1058 snap 211 seq_num 1058) dirty exists 2014-07-01 17:03:15,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: finishing write tid 1 to mira0435601-481 2014-07-01 17:03:15,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: finishing write tid 1 to mira0435601-244 2014-07-01 17:03:15,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: finishing write tid 3 to mira0435601-481 2014-07-01 17:03:15,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: finishing write tid 2 to mira0435601-244 2014-07-01 17:03:15,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 1 to mira0435601-59 2014-07-01 17:03:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 2 to mira0435601-59 2014-07-01 17:03:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 3 to mira0435601-59 2014-07-01 17:03:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: finishing write tid 3 to mira0435601-244 2014-07-01 17:03:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 5 to mira0435601-59 2014-07-01 17:03:15,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: finishing write tid 5 to mira0435601-244 2014-07-01 17:03:15,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 1 to mira0435601-370 2014-07-01 17:03:15,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: finishing write tid 4 to mira0435601-481 2014-07-01 17:03:15,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 160 (ObjNum 1979 snap 0 seq_num 1979) dirty exists 2014-07-01 17:03:15,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 2 to mira0435601-370 2014-07-01 17:03:15,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: done (8 left) 2014-07-01 17:03:15,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3195: done (7 left) 2014-07-01 17:03:15,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: done (6 left) 2014-07-01 17:03:15,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: done (5 left) 2014-07-01 17:03:15,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: done (4 left) 2014-07-01 17:03:15,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: delete oid 210 current snap is 0 2014-07-01 17:03:15,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 6 to mira0435601-59 2014-07-01 17:03:15,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 275 (ObjNum 1981 snap 0 seq_num 1981) dirty exists 2014-07-01 17:03:15,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 3 to mira0435601-370 2014-07-01 17:03:15,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: finishing write tid 1 to mira0435601-223 2014-07-01 17:03:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: done (4 left) 2014-07-01 17:03:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: done (3 left) 2014-07-01 17:03:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 17:03:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: seq_num 1989 ranges {7516160=589824,8105984=593920,8699904=589824,9289728=20480} 2014-07-01 17:03:17,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2029: done (6 left) 2014-07-01 17:03:17,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2030: done (5 left) 2014-07-01 17:03:17,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2031: done (4 left) 2014-07-01 17:03:17,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:2032: read oid 320 snap -1 2014-07-01 17:03:17,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:2033: read oid 22 snap -1 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: write oid 315 current snap is 211 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: seq_num 1060 ranges {494501=665339,1682784=411717} 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: writing mira0437250-315 from 494501 to 1159840 tid 1 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: writing mira0437250-315 from 1682784 to 2094501 tid 2 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: write oid 191 current snap is 211 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: seq_num 1061 ranges {686883=672215,2044469=463955,3170669=591045,3886882=1} 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: writing mira0437250-191 from 686883 to 1359098 tid 1 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: writing mira0437250-191 from 2044469 to 2508424 tid 2 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: writing mira0435601-311 from 7516160 to 8105984 tid 1 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: writing mira0435601-311 from 8105984 to 8699904 tid 2 2014-07-01 17:03:17,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: writing mira0435601-311 from 8699904 to 9289728 tid 3 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: writing mira0435601-311 from 9289728 to 9310208 tid 4 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3202: delete oid 255 current snap is 0 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 4 to mira0435601-370 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: finishing write tid 3 to mira0435601-223 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 6 to mira0435601-370 2014-07-01 17:03:17,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: finishing write tid 6 to mira0435601-244 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 219 (ObjNum 1982 snap 0 seq_num 1982) dirty exists 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: finishing write tid 4 to mira0435601-223 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 218 (ObjNum 1984 snap 0 seq_num 1984) dirty exists 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: finishing write tid 7 to mira0435601-370 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 166 (ObjNum 1986 snap 0 seq_num 1986) dirty exists 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: writing mira0437250-191 from 3170669 to 3761714 tid 3 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: writing mira0437250-191 from 3886882 to 3886883 tid 4 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: write oid 173 current snap is 211 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: seq_num 1062 ranges {782112=645044,2158601=534150,3372918=209194} 2014-07-01 17:03:17,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: writing mira0437250-173 from 782112 to 1427156 tid 1 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: writing mira0437250-173 from 2158601 to 2692751 tid 2 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: writing mira0437250-173 from 3372918 to 3582112 tid 3 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: rollback oid 491 current snap is 211 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 491 to 209 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 1 to mira0437250-315 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 2 to mira0437250-315 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 4 to mira0437250-315 2014-07-01 17:03:17,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: finishing write tid 5 to mira0437250-315 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 359 (ObjNum 1060 snap 211 seq_num 1060) dirty exists 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2032: expect (ObjNum 319 snap 0 seq_num 319) 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: done (4 left) 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: done (3 left) 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: done (2 left) 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3202: done (1 left) 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 263 current snap is 0 2014-07-01 17:03:17,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: seq_num 1991 ranges {3194880=733184,3928064=405504,4333568=397312} 2014-07-01 17:03:17,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: writing mira0435601-263 from 3194880 to 3928064 tid 1 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: writing mira0435601-263 from 3928064 to 4333568 tid 2 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: writing mira0435601-263 from 4333568 to 4730880 tid 3 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: read oid 233 snap -1 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 1 to mira0435601-311 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 2 to mira0435601-311 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 3 to mira0435601-311 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 4 to mira0435601-311 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 6 to mira0435601-311 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3205: read oid 268 snap -1 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3206: read oid 298 snap -1 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: seq_num 1992 ranges {1142784=360448} 2014-07-01 17:03:17,341.341 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: writing mira0435601-323 from 1142784 to 1503232 tid 1 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3208: read oid 320 snap -1 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 185 current snap is 0 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: seq_num 1993 ranges {544768=684032,1228800=401408} 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: writing mira0435601-185 from 544768 to 1228800 tid 1 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: writing mira0435601-185 from 1228800 to 1630208 tid 2 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 326 current snap is 0 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: seq_num 1994 ranges {1388544=552960,1941504=401408} 2014-07-01 17:03:17,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: writing mira0435601-326 from 1388544 to 1941504 tid 1 2014-07-01 17:03:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: writing mira0435601-326 from 1941504 to 2342912 tid 2 2014-07-01 17:03:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 483 current snap is 0 2014-07-01 17:03:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: seq_num 1995 ranges {3444736=581632,4026368=634880,4661248=532480,5193728=438272} 2014-07-01 17:03:17,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: writing mira0435601-483 from 3444736 to 4026368 tid 1 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: writing mira0435601-483 from 4026368 to 4661248 tid 2 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: writing mira0435601-483 from 4661248 to 5193728 tid 3 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: writing mira0435601-483 from 5193728 to 5632000 tid 4 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 290 current snap is 0 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: seq_num 1996 ranges {0=454656,454656=618496,1073152=409600,1482752=577536} 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: writing mira0435601-290 from 0 to 454656 tid 1 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: writing mira0435601-290 from 454656 to 1073152 tid 2 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: writing mira0435601-290 from 1073152 to 1482752 tid 3 2014-07-01 17:03:17,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: writing mira0435601-290 from 1482752 to 2060288 tid 4 2014-07-01 17:03:17,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3213: read oid 380 snap -1 2014-07-01 17:03:17,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: expect (ObjNum 1811 snap 0 seq_num 1811) 2014-07-01 17:03:18,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2033: expect (ObjNum 21 snap 0 seq_num 21) 2014-07-01 17:03:18,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 1 to mira0437250-191 2014-07-01 17:03:18,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: expect (ObjNum 969 snap 178 seq_num 969) 2014-07-01 17:03:19,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3208: expect (ObjNum 952 snap 0 seq_num 952) 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 2 to mira0437250-191 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 782 (ObjNum 884 snap 150 seq_num 884) dirty exists 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 3 to mira0437250-191 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 4 to mira0437250-191 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 6 to mira0437250-191 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: finishing write tid 7 to mira0437250-191 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 682 (ObjNum 1061 snap 211 seq_num 1061) dirty exists 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 491 v 784 (ObjNum 490 snap 0 seq_num 490) dirty exists 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: finishing write tid 1 to mira0437250-173 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: finishing write tid 2 to mira0437250-173 2014-07-01 17:03:19,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:2026: done (9 left) 2014-07-01 17:03:19,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:2027: done (8 left) 2014-07-01 17:03:19,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:2028: done (7 left) 2014-07-01 17:03:19,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:2032: done (6 left) 2014-07-01 17:03:19,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:2033: done (5 left) 2014-07-01 17:03:19,432.432 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:19,432.432 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:19,432.432 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:03:19,432.432 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:03:19,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2034: done (4 left) 2014-07-01 17:03:19,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2035: done (3 left) 2014-07-01 17:03:19,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2037: done (2 left) 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: write oid 187 current snap is 211 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: seq_num 1063 ranges {478366=768272,1989085=487993,3174144=104222} 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: writing mira0437250-187 from 478366 to 1246638 tid 1 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: writing mira0437250-187 from 1989085 to 2477078 tid 2 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: writing mira0437250-187 from 3174144 to 3278366 tid 3 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2039: snap_create 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: finishing write tid 3 to mira0437250-173 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: finishing write tid 5 to mira0437250-173 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: finishing write tid 6 to mira0437250-173 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 1374 (ObjNum 1062 snap 211 seq_num 1062) dirty exists 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 1 to mira0437250-122 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 3 to mira0437250-122 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: finishing write tid 4 to mira0437250-122 2014-07-01 17:03:19,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 487 (ObjNum 1055 snap 210 seq_num 1055) dirty exists 2014-07-01 17:03:19,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2019: done (3 left) 2014-07-01 17:03:19,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2036: done (2 left) 2014-07-01 17:03:19,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2039: done (1 left) 2014-07-01 17:03:19,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2040: rollback oid 410 current snap is 212 2014-07-01 17:03:19,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 410 to 199 2014-07-01 17:03:19,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: read oid 295 snap 180 2014-07-01 17:03:19,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2042: delete oid 193 current snap is 212 2014-07-01 17:03:19,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: expect (ObjNum 626 snap 53 seq_num 626) 2014-07-01 17:03:19,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2041: done (3 left) 2014-07-01 17:03:19,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2042: done (2 left) 2014-07-01 17:03:19,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: write oid 121 current snap is 212 2014-07-01 17:03:19,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: seq_num 1065 ranges {483354=596553,1283353=1} 2014-07-01 17:03:19,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: writing mira0437250-121 from 483354 to 1079907 tid 1 2014-07-01 17:03:20,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: writing mira0437250-121 from 1283353 to 1283354 tid 2 2014-07-01 17:03:20,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: write oid 470 current snap is 212 2014-07-01 17:03:20,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: seq_num 1066 ranges {579607=400000} 2014-07-01 17:03:20,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: writing mira0437250-470 from 579607 to 979607 tid 1 2014-07-01 17:03:20,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2045: snap_create 2014-07-01 17:03:20,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: finishing write tid 1 to mira0437250-470 2014-07-01 17:03:20,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: finishing write tid 3 to mira0437250-470 2014-07-01 17:03:20,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: finishing write tid 4 to mira0437250-470 2014-07-01 17:03:20,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 735 (ObjNum 1066 snap 212 seq_num 1066) dirty exists 2014-07-01 17:03:20,223.223 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:03:20,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2044: done (4 left) 2014-07-01 17:03:20,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2045: done (3 left) 2014-07-01 17:03:20,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2046: snap_remove snap 192 2014-07-01 17:03:20,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: finishing write tid 1 to mira0437250-121 2014-07-01 17:03:20,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: finishing write tid 2 to mira0437250-121 2014-07-01 17:03:20,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: finishing write tid 4 to mira0437250-121 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 788 (ObjNum 865 snap 145 seq_num 865) dirty exists 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: finishing write tid 1 to mira0437250-187 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: finishing write tid 2 to mira0437250-187 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: finishing write tid 3 to mira0437250-187 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: finishing write tid 5 to mira0437250-187 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: finishing write tid 6 to mira0437250-187 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 1189 (ObjNum 1063 snap 211 seq_num 1063) dirty exists 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: finishing write tid 5 to mira0437250-121 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 873 (ObjNum 1065 snap 212 seq_num 1065) dirty exists 2014-07-01 17:03:20,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3213: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 17:03:21,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: finishing write tid 1 to mira0435601-326 2014-07-01 17:03:21,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: finishing write tid 2 to mira0435601-326 2014-07-01 17:03:21,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: finishing write tid 4 to mira0435601-326 2014-07-01 17:03:21,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: finishing write tid 1 to mira0435601-263 2014-07-01 17:03:21,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 1 to mira0435601-290 2014-07-01 17:03:21,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: finishing write tid 2 to mira0435601-263 2014-07-01 17:03:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 2 to mira0435601-290 2014-07-01 17:03:21,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: finishing write tid 3 to mira0435601-263 2014-07-01 17:03:21,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 3 to mira0435601-290 2014-07-01 17:03:21,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: finishing write tid 1 to mira0435601-323 2014-07-01 17:03:21,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: finishing write tid 5 to mira0435601-263 2014-07-01 17:03:21,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 4 to mira0435601-290 2014-07-01 17:03:21,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: finishing write tid 3 to mira0435601-323 2014-07-01 17:03:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 6 to mira0435601-290 2014-07-01 17:03:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: finishing write tid 5 to mira0435601-326 2014-07-01 17:03:21,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 129 (ObjNum 1994 snap 0 seq_num 1994) dirty exists 2014-07-01 17:03:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: finishing write tid 1 to mira0435601-185 2014-07-01 17:03:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: done (11 left) 2014-07-01 17:03:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3205: done (10 left) 2014-07-01 17:03:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3206: done (9 left) 2014-07-01 17:03:21,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3208: done (8 left) 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: done (7 left) 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3213: done (6 left) 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3214: read oid 490 snap -1 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: finishing write tid 7 to mira0435601-311 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 280 (ObjNum 1989 snap 0 seq_num 1989) dirty exists 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: finishing write tid 2 to mira0435601-185 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: finishing write tid 4 to mira0435601-185 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: finishing write tid 7 to mira0435601-290 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 221 (ObjNum 1996 snap 0 seq_num 1996) dirty exists 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: finishing write tid 4 to mira0435601-323 2014-07-01 17:03:21,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 208 (ObjNum 1992 snap 0 seq_num 1992) dirty exists 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: finishing write tid 6 to mira0435601-263 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 278 (ObjNum 1991 snap 0 seq_num 1991) dirty exists 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: finishing write tid 5 to mira0435601-185 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 162 (ObjNum 1993 snap 0 seq_num 1993) dirty exists 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 1 to mira0435601-483 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: done (6 left) 2014-07-01 17:03:21,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: done (5 left) 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: done (4 left) 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: done (3 left) 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: done (2 left) 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3215: read oid 436 snap -1 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 2 to mira0435601-483 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 3 to mira0435601-483 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 4 to mira0435601-483 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 6 to mira0435601-483 2014-07-01 17:03:21,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 7 to mira0435601-483 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 185 (ObjNum 1995 snap 0 seq_num 1995) dirty exists 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: done (2 left) 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 500 current snap is 0 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: seq_num 1997 ranges {1720320=479232,2199552=401408} 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: writing mira0435601-500 from 1720320 to 2199552 tid 1 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: writing mira0435601-500 from 2199552 to 2600960 tid 2 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 421 current snap is 0 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: seq_num 1998 ranges {4796416=802816,5599232=786432,6385664=16384} 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: writing mira0435601-421 from 4796416 to 5599232 tid 1 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: writing mira0435601-421 from 5599232 to 6385664 tid 2 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: writing mira0435601-421 from 6385664 to 6402048 tid 3 2014-07-01 17:03:21,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 179 current snap is 0 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: seq_num 1999 ranges {4628480=671744,5300224=614400,5914624=188416} 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: writing mira0435601-179 from 4628480 to 5300224 tid 1 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2038: done (3 left) 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2040: done (2 left) 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2043: done (1 left) 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2046: done (0 left) 2014-07-01 17:03:21,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: snap_remove snap 190 2014-07-01 17:03:21,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: writing mira0435601-179 from 5300224 to 5914624 tid 2 2014-07-01 17:03:21,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: writing mira0435601-179 from 5914624 to 6103040 tid 3 2014-07-01 17:03:21,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 339 current snap is 0 2014-07-01 17:03:21,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: seq_num 2000 ranges {0=299008} 2014-07-01 17:03:21,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: writing mira0435601-339 from 0 to 299008 tid 1 2014-07-01 17:03:21,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 30 current snap is 0 2014-07-01 17:03:21,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: seq_num 2001 ranges {4292608=172032} 2014-07-01 17:03:21,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: writing mira0435601-30 from 4292608 to 4464640 tid 1 2014-07-01 17:03:21,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: read oid 37 snap -1 2014-07-01 17:03:21,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 1 to mira0435601-500 2014-07-01 17:03:21,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3214: expect (ObjNum 879 snap 0 seq_num 879) 2014-07-01 17:03:22,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2047: done (0 left) 2014-07-01 17:03:22,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: rollback oid 154 current snap is 213 2014-07-01 17:03:22,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 154 to 184 2014-07-01 17:03:22,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: write oid 293 current snap is 213 2014-07-01 17:03:22,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: seq_num 1067 ranges {771989=691585,1954221=671622,2771988=1} 2014-07-01 17:03:22,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0437250-293 from 771989 to 1463574 tid 1 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0437250-293 from 1954221 to 2625843 tid 2 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: writing mira0437250-293 from 2771988 to 2771989 tid 3 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: rollback oid 126 current snap is 213 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 126 to 180 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2051: rollback oid 57 current snap is 213 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 57 to 181 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: snap_remove snap 168 2014-07-01 17:03:22,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 0 (ObjNum 16743968 snap 0 seq_num 15816016) dirty dne 2014-07-01 17:03:23,446.446 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:03:23,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 1 to mira0437250-293 2014-07-01 17:03:23,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 2 to mira0437250-293 2014-07-01 17:03:23,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2050: done (4 left) 2014-07-01 17:03:23,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2052: done (3 left) 2014-07-01 17:03:23,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: write oid 86 current snap is 213 2014-07-01 17:03:23,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: seq_num 1068 ranges {468243=663390,1784706=493241,2468242=1} 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: writing mira0437250-86 from 468243 to 1131633 tid 1 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: writing mira0437250-86 from 1784706 to 2277947 tid 2 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: writing mira0437250-86 from 2468242 to 2468243 tid 3 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: snap_remove snap 208 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 3 to mira0437250-293 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 5 to mira0437250-293 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: finishing write tid 6 to mira0437250-293 2014-07-01 17:03:23,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 869 (ObjNum 1067 snap 213 seq_num 1067) dirty exists 2014-07-01 17:03:23,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 642 (ObjNum 153 snap 0 seq_num 153) dirty exists 2014-07-01 17:03:23,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 1265 (ObjNum 955 snap 175 seq_num 955) dirty exists 2014-07-01 17:03:23,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: finishing write tid 1 to mira0437250-86 2014-07-01 17:03:23,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: finishing write tid 2 to mira0437250-86 2014-07-01 17:03:23,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: finishing write tid 3 to mira0437250-86 2014-07-01 17:03:23,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: finishing write tid 5 to mira0437250-86 2014-07-01 17:03:23,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: finishing write tid 6 to mira0437250-86 2014-07-01 17:03:23,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 468 (ObjNum 1068 snap 213 seq_num 1068) dirty exists 2014-07-01 17:03:24,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 2 to mira0435601-500 2014-07-01 17:03:24,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 4 to mira0435601-500 2014-07-01 17:03:24,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3214: done (7 left) 2014-07-01 17:03:24,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 126 current snap is 0 2014-07-01 17:03:24,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: seq_num 2002 ranges {1851392=491520,2342912=401408} 2014-07-01 17:03:24,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: writing mira0435601-126 from 1851392 to 2342912 tid 1 2014-07-01 17:03:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: writing mira0435601-126 from 2342912 to 2744320 tid 2 2014-07-01 17:03:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 282 current snap is 0 2014-07-01 17:03:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: seq_num 2003 ranges {6356992=520192,6877184=593920,7471104=630784,8101888=380928} 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: writing mira0435601-282 from 6356992 to 6877184 tid 1 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: writing mira0435601-282 from 6877184 to 7471104 tid 2 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: writing mira0435601-282 from 7471104 to 8101888 tid 3 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: writing mira0435601-282 from 8101888 to 8482816 tid 4 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 70 current snap is 0 2014-07-01 17:03:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: seq_num 2004 ranges {3006464=794624,3801088=581632,4382720=622592} 2014-07-01 17:03:24,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: writing mira0435601-70 from 3006464 to 3801088 tid 1 2014-07-01 17:03:24,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: writing mira0435601-70 from 3801088 to 4382720 tid 2 2014-07-01 17:03:24,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: writing mira0435601-70 from 4382720 to 5005312 tid 3 2014-07-01 17:03:24,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: read oid 206 snap -1 2014-07-01 17:03:24,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 5 to mira0435601-500 2014-07-01 17:03:24,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 500 v 230 (ObjNum 1997 snap 0 seq_num 1997) dirty exists 2014-07-01 17:03:24,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3215: expect (ObjNum 1653 snap 0 seq_num 1653) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2048: done (4 left) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2049: done (3 left) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2051: done (2 left) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2053: done (1 left) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2054: done (0 left) 2014-07-01 17:03:24,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2055: delete oid 270 current snap is 213 2014-07-01 17:03:24,667.667 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:03:24,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2055: done (0 left) 2014-07-01 17:03:24,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2056: delete oid 83 current snap is 213 2014-07-01 17:03:24,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2056: done (0 left) 2014-07-01 17:03:24,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: read oid 266 snap 185 2014-07-01 17:03:24,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: snap_create 2014-07-01 17:03:24,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: expect (ObjNum 265 snap 0 seq_num 265) 2014-07-01 17:03:25,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2057: done (1 left) 2014-07-01 17:03:25,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2058: done (0 left) 2014-07-01 17:03:25,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2059: delete oid 56 current snap is 214 2014-07-01 17:03:25,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2059: done (0 left) 2014-07-01 17:03:25,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: write oid 316 current snap is 214 2014-07-01 17:03:25,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: seq_num 1072 ranges {663690=739151,2053714=408070,3062100=710604,3863689=1} 2014-07-01 17:03:25,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: writing mira0437250-316 from 663690 to 1402841 tid 1 2014-07-01 17:03:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: writing mira0437250-316 from 2053714 to 2461784 tid 2 2014-07-01 17:03:25,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: writing mira0437250-316 from 3062100 to 3772704 tid 3 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: writing mira0437250-316 from 3863689 to 3863690 tid 4 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: write oid 2 current snap is 214 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: seq_num 1073 ranges {724872=400000} 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: writing mira0437250-2 from 724872 to 1124872 tid 1 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: read oid 63 snap -1 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 1 to mira0437250-316 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 2 to mira0437250-316 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 3 to mira0437250-316 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 4 to mira0437250-316 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 6 to mira0437250-316 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: read oid 273 snap 203 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: read oid 411 snap -1 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: write oid 263 current snap is 214 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: seq_num 1074 ranges {668677=717832,1844858=23819} 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: writing mira0437250-263 from 668677 to 1386509 tid 1 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: writing mira0437250-263 from 1844858 to 1868677 tid 2 2014-07-01 17:03:25,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: write oid 206 current snap is 214 2014-07-01 17:03:25,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: seq_num 1075 ranges {764931=689636,1564930=1} 2014-07-01 17:03:25,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: writing mira0437250-206 from 764931 to 1454567 tid 1 2014-07-01 17:03:25,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: writing mira0437250-206 from 1564930 to 1564931 tid 2 2014-07-01 17:03:25,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: read oid 403 snap -1 2014-07-01 17:03:25,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: expect (ObjNum 272 snap 0 seq_num 272) 2014-07-01 17:03:25,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: expect (ObjNum 1023 snap 201 seq_num 1023) 2014-07-01 17:03:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 1 to mira0437250-2 2014-07-01 17:03:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 3 to mira0437250-2 2014-07-01 17:03:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: finishing write tid 4 to mira0437250-2 2014-07-01 17:03:26,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 2 v 523 (ObjNum 1073 snap 214 seq_num 1073) dirty exists 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 1 to mira0437250-263 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 2 to mira0437250-263 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 4 to mira0437250-263 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: finishing write tid 5 to mira0437250-263 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 263 v 887 (ObjNum 1074 snap 214 seq_num 1074) dirty exists 2014-07-01 17:03:26,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: expect (ObjNum 735 snap 100 seq_num 735) 2014-07-01 17:03:28,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: finishing write tid 7 to mira0437250-316 2014-07-01 17:03:28,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 587 (ObjNum 1072 snap 214 seq_num 1072) dirty exists 2014-07-01 17:03:28,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: finishing write tid 1 to mira0437250-206 2014-07-01 17:03:28,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: finishing write tid 2 to mira0437250-206 2014-07-01 17:03:28,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: finishing write tid 4 to mira0437250-206 2014-07-01 17:03:28,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2060: done (7 left) 2014-07-01 17:03:28,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2061: done (6 left) 2014-07-01 17:03:28,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2062: done (5 left) 2014-07-01 17:03:28,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2063: done (4 left) 2014-07-01 17:03:28,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2064: done (3 left) 2014-07-01 17:03:28,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2065: done (2 left) 2014-07-01 17:03:28,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: snap_remove snap 181 2014-07-01 17:03:28,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: expect (ObjNum 402 snap 0 seq_num 402) 2014-07-01 17:03:28,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: finishing write tid 1 to mira0435601-30 2014-07-01 17:03:28,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: finishing write tid 3 to mira0435601-30 2014-07-01 17:03:28,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: finishing write tid 4 to mira0435601-30 2014-07-01 17:03:28,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 30 v 168 (ObjNum 2001 snap 0 seq_num 2001) dirty exists 2014-07-01 17:03:28,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: expect (ObjNum 1585 snap 0 seq_num 1585) 2014-07-01 17:03:28,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: finishing write tid 5 to mira0437250-206 2014-07-01 17:03:28,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 1144 (ObjNum 1075 snap 214 seq_num 1075) dirty exists 2014-07-01 17:03:28,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2066: done (2 left) 2014-07-01 17:03:28,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2067: done (1 left) 2014-07-01 17:03:28,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:2068: done (0 left) 2014-07-01 17:03:28,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: read oid 107 snap -1 2014-07-01 17:03:28,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: snap_create 2014-07-01 17:03:29,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: expect (ObjNum 106 snap 0 seq_num 106) 2014-07-01 17:03:29,687.687 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:29,687.687 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:29,688.688 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 5 to 0.449601623619 2014-07-01 17:03:29,688.688 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 0.449601623619' 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: finishing write tid 1 to mira0435601-421 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: finishing write tid 2 to mira0435601-421 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: finishing write tid 3 to mira0435601-421 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: finishing write tid 5 to mira0435601-421 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: finishing write tid 1 to mira0435601-179 2014-07-01 17:03:29,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: finishing write tid 2 to mira0435601-179 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: finishing write tid 3 to mira0435601-179 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: finishing write tid 5 to mira0435601-179 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: finishing write tid 1 to mira0435601-339 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: finishing write tid 3 to mira0435601-339 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: finishing write tid 6 to mira0435601-421 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 316 (ObjNum 1998 snap 0 seq_num 1998) dirty exists 2014-07-01 17:03:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: finishing write tid 6 to mira0435601-179 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 179 v 225 (ObjNum 1999 snap 0 seq_num 1999) dirty exists 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: finishing write tid 4 to mira0435601-339 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 289 (ObjNum 2000 snap 0 seq_num 2000) dirty exists 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 1 to mira0435601-126 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3215: done (10 left) 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: done (9 left) 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: done (8 left) 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: done (7 left) 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: done (6 left) 2014-07-01 17:03:29,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: done (5 left) 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: done (4 left) 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 286 current snap is 0 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: seq_num 2005 ranges {2850816=663552,3514368=606208,4120576=528384,4648960=69632} 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: writing mira0435601-286 from 2850816 to 3514368 tid 1 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: writing mira0435601-286 from 3514368 to 4120576 tid 2 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: writing mira0435601-286 from 4120576 to 4648960 tid 3 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: writing mira0435601-286 from 4648960 to 4718592 tid 4 2014-07-01 17:03:29,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3227: read oid 131 snap -1 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 2 to mira0435601-126 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 4 to mira0435601-126 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 5 to mira0435601-126 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 304 (ObjNum 2002 snap 0 seq_num 2002) dirty exists 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 1 to mira0435601-282 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 2 to mira0435601-282 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 3 to mira0435601-282 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 4 to mira0435601-282 2014-07-01 17:03:29,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 6 to mira0435601-282 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: finishing write tid 7 to mira0435601-282 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 328 (ObjNum 2003 snap 0 seq_num 2003) dirty exists 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: finishing write tid 1 to mira0435601-70 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: finishing write tid 2 to mira0435601-70 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: finishing write tid 3 to mira0435601-70 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: finishing write tid 5 to mira0435601-70 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: finishing write tid 6 to mira0435601-70 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 70 v 308 (ObjNum 2004 snap 0 seq_num 2004) dirty exists 2014-07-01 17:03:29,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: expect (ObjNum 1823 snap 0 seq_num 1823) 2014-07-01 17:03:30,740.740 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 0.449602 (8294652) 2014-07-01 17:03:30,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2069: done (1 left) 2014-07-01 17:03:30,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2070: done (0 left) 2014-07-01 17:03:30,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: read oid 441 snap -1 2014-07-01 17:03:30,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2072: delete oid 305 current snap is 215 2014-07-01 17:03:30,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: expect (ObjNum 575 snap 28 seq_num 575) 2014-07-01 17:03:32,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2071: done (1 left) 2014-07-01 17:03:32,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2072: done (0 left) 2014-07-01 17:03:32,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: snap_remove snap 209 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 1 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 2 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 3 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 4 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 6 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: finishing write tid 7 to mira0435601-286 2014-07-01 17:03:32,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 361 (ObjNum 2005 snap 0 seq_num 2005) dirty exists 2014-07-01 17:03:32,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3227: expect (ObjNum 815 snap 0 seq_num 815) 2014-07-01 17:03:32,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2073: done (0 left) 2014-07-01 17:03:32,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: read oid 269 snap -1 2014-07-01 17:03:32,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2075: snap_create 2014-07-01 17:03:33,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: expect (ObjNum 268 snap 0 seq_num 268) 2014-07-01 17:03:33,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: done (5 left) 2014-07-01 17:03:33,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: done (4 left) 2014-07-01 17:03:33,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: done (3 left) 2014-07-01 17:03:33,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: done (2 left) 2014-07-01 17:03:33,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: done (1 left) 2014-07-01 17:03:33,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3227: done (0 left) 2014-07-01 17:03:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 23 current snap is 0 2014-07-01 17:03:33,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: seq_num 2006 ranges {0=692224} 2014-07-01 17:03:33,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: writing mira0435601-23 from 0 to 692224 tid 1 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3229: read oid 175 snap -1 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 1 current snap is 0 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: seq_num 2007 ranges {0=614400,614400=417792,1032192=385024} 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: writing mira0435601-1 from 0 to 614400 tid 1 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: writing mira0435601-1 from 614400 to 1032192 tid 2 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: writing mira0435601-1 from 1032192 to 1417216 tid 3 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: read oid 40 snap -1 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3232: read oid 183 snap -1 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 95 current snap is 0 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: seq_num 2008 ranges {0=483328,483328=405504,888832=397312} 2014-07-01 17:03:33,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: writing mira0435601-95 from 0 to 483328 tid 1 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: writing mira0435601-95 from 483328 to 888832 tid 2 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: writing mira0435601-95 from 888832 to 1286144 tid 3 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: seq_num 2009 ranges {3428352=110592} 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: writing mira0435601-228 from 3428352 to 3538944 tid 1 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: read oid 199 snap -1 2014-07-01 17:03:33,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 17:03:34,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: finishing write tid 1 to mira0435601-23 2014-07-01 17:03:34,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: finishing write tid 3 to mira0435601-23 2014-07-01 17:03:34,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: finishing write tid 1 to mira0435601-95 2014-07-01 17:03:34,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: finishing write tid 2 to mira0435601-95 2014-07-01 17:03:34,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: finishing write tid 3 to mira0435601-95 2014-07-01 17:03:34,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: finishing write tid 5 to mira0435601-95 2014-07-01 17:03:34,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: finishing write tid 1 to mira0435601-228 2014-07-01 17:03:34,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: finishing write tid 3 to mira0435601-228 2014-07-01 17:03:34,151.151 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: finishing write tid 4 to mira0435601-23 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 206 (ObjNum 2006 snap 0 seq_num 2006) dirty exists 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: finishing write tid 4 to mira0435601-228 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 170 (ObjNum 2009 snap 0 seq_num 2009) dirty exists 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: finishing write tid 6 to mira0435601-95 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 312 (ObjNum 2008 snap 0 seq_num 2008) dirty exists 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: done (7 left) 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: done (6 left) 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: done (5 left) 2014-07-01 17:03:34,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: done (4 left) 2014-07-01 17:03:34,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 277 current snap is 0 2014-07-01 17:03:34,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: seq_num 2010 ranges {2220032=540672,2760704=569344,3330048=233472} 2014-07-01 17:03:34,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: writing mira0435601-277 from 2220032 to 2760704 tid 1 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: writing mira0435601-277 from 2760704 to 3330048 tid 2 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: writing mira0435601-277 from 3330048 to 3563520 tid 3 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 240 current snap is 0 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: seq_num 2011 ranges {3166208=704512} 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: writing mira0435601-240 from 3166208 to 3870720 tid 1 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 17:03:34,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: seq_num 2012 ranges {8708096=733184,9441280=786432,10227712=417792} 2014-07-01 17:03:34,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: writing mira0435601-394 from 8708096 to 9441280 tid 1 2014-07-01 17:03:34,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: writing mira0435601-394 from 9441280 to 10227712 tid 2 2014-07-01 17:03:34,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: writing mira0435601-394 from 10227712 to 10645504 tid 3 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 64 current snap is 0 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: seq_num 2013 ranges {3297280=761856} 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: writing mira0435601-64 from 3297280 to 4059136 tid 1 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: read oid 464 snap -1 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3229: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2074: done (1 left) 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2075: done (0 left) 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2076: rollback oid 133 current snap is 216 2014-07-01 17:03:34,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 133 to 188 2014-07-01 17:03:34,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: snap_remove snap 180 2014-07-01 17:03:34,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 605 (ObjNum 132 snap 0 seq_num 132) dirty exists 2014-07-01 17:03:34,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2076: done (1 left) 2014-07-01 17:03:34,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2077: done (0 left) 2014-07-01 17:03:34,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: write oid 83 current snap is 216 2014-07-01 17:03:34,771.771 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: seq_num 1077 ranges {653567=669341,1884407=510470,3053566=1} 2014-07-01 17:03:34,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: writing mira0437250-83 from 653567 to 1322908 tid 1 2014-07-01 17:03:34,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: writing mira0437250-83 from 1884407 to 2394877 tid 2 2014-07-01 17:03:34,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: writing mira0437250-83 from 3053566 to 3053567 tid 3 2014-07-01 17:03:34,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: snap_create 2014-07-01 17:03:35,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: finishing write tid 1 to mira0435601-1 2014-07-01 17:03:35,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: finishing write tid 2 to mira0435601-1 2014-07-01 17:03:35,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:3232: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 17:03:35,760.760 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:35,761.761 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:35,761.761 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_2 2014-07-01 17:03:35,762.762 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:03:35,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2079: done (1 left) 2014-07-01 17:03:35,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: rollback oid 130 current snap is 217 2014-07-01 17:03:35,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 130 to 207 2014-07-01 17:03:35,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2081: rollback oid 9 current snap is 217 2014-07-01 17:03:35,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 9 to 158 2014-07-01 17:03:35,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: delete oid 328 current snap is 217 2014-07-01 17:03:35,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2082: done (3 left) 2014-07-01 17:03:35,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: read oid 215 snap -1 2014-07-01 17:03:35,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: snap_remove snap 213 2014-07-01 17:03:35,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 1142 (ObjNum 8 snap 0 seq_num 8) dirty exists 2014-07-01 17:03:35,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: expect (ObjNum 979 snap 181 seq_num 979) 2014-07-01 17:03:35,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 1 to mira0435601-277 2014-07-01 17:03:35,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 2 to mira0435601-277 2014-07-01 17:03:35,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 3 to mira0435601-277 2014-07-01 17:03:35,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 5 to mira0435601-277 2014-07-01 17:03:35,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 6 to mira0435601-277 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 155 (ObjNum 2010 snap 0 seq_num 2010) dirty exists 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: finishing write tid 3 to mira0435601-1 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3229: done (8 left) 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3232: done (7 left) 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: done (6 left) 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 360 current snap is 0 2014-07-01 17:03:35,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: seq_num 2014 ranges {32768=475136,507904=552960,1060864=651264} 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: writing mira0435601-360 from 32768 to 507904 tid 1 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: writing mira0435601-360 from 507904 to 1060864 tid 2 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: writing mira0435601-360 from 1060864 to 1712128 tid 3 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: read oid 237 snap -1 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: finishing write tid 5 to mira0435601-1 2014-07-01 17:03:36,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: expect (ObjNum 1591 snap 0 seq_num 1591) 2014-07-01 17:03:36,508.508 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:03:36,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: finishing write tid 1 to mira0435601-240 2014-07-01 17:03:36,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: finishing write tid 3 to mira0435601-240 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: finishing write tid 4 to mira0435601-240 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 240 v 164 (ObjNum 2011 snap 0 seq_num 2011) dirty exists 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: finishing write tid 6 to mira0435601-1 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 200 (ObjNum 2007 snap 0 seq_num 2007) dirty exists 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: finishing write tid 1 to mira0435601-64 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: done (7 left) 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: done (6 left) 2014-07-01 17:03:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: done (5 left) 2014-07-01 17:03:36,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 17:03:36,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: seq_num 2015 ranges {6340608=503808} 2014-07-01 17:03:36,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: writing mira0435601-435 from 6340608 to 6844416 tid 1 2014-07-01 17:03:36,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: seq_num 2016 ranges {0=180224} 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: writing mira0435601-434 from 0 to 180224 tid 1 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3245: read oid 418 snap -1 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: finishing write tid 3 to mira0435601-64 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 1 to mira0435601-394 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 2 to mira0435601-394 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 3 to mira0435601-394 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 5 to mira0435601-394 2014-07-01 17:03:36,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: finishing write tid 4 to mira0435601-64 2014-07-01 17:03:36,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 232 (ObjNum 2013 snap 0 seq_num 2013) dirty exists 2014-07-01 17:03:36,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 6 to mira0435601-394 2014-07-01 17:03:36,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 205 (ObjNum 2012 snap 0 seq_num 2012) dirty exists 2014-07-01 17:03:36,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: expect (ObjNum 1530 snap 0 seq_num 1530) 2014-07-01 17:03:37,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 1 to mira0437250-83 2014-07-01 17:03:37,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 643 (ObjNum 994 snap 189 seq_num 994) dirty exists 2014-07-01 17:03:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 2 to mira0437250-83 2014-07-01 17:03:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 3 to mira0437250-83 2014-07-01 17:03:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 5 to mira0437250-83 2014-07-01 17:03:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: finishing write tid 6 to mira0437250-83 2014-07-01 17:03:37,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 480 (ObjNum 1077 snap 216 seq_num 1077) dirty exists 2014-07-01 17:03:37,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2078: done (4 left) 2014-07-01 17:03:37,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2080: done (3 left) 2014-07-01 17:03:37,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2081: done (2 left) 2014-07-01 17:03:37,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2083: done (1 left) 2014-07-01 17:03:37,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:2084: done (0 left) 2014-07-01 17:03:37,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: write oid 494 current snap is 217 2014-07-01 17:03:37,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: seq_num 1079 ranges {8953=1} 2014-07-01 17:03:37,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: writing mira0437250-494 from 8953 to 8954 tid 1 2014-07-01 17:03:37,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:2086: rollback oid 61 current snap is 217 2014-07-01 17:03:37,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 61 to 211 2014-07-01 17:03:37,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2087: snap_remove snap 212 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 0 (ObjNum 33134216 snap 0 seq_num 16922016) dirty dne 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: finishing write tid 1 to mira0437250-494 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: finishing write tid 3 to mira0437250-494 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: finishing write tid 4 to mira0437250-494 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 616 (ObjNum 1079 snap 217 seq_num 1079) dirty exists 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2085: done (2 left) 2014-07-01 17:03:37,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2086: done (1 left) 2014-07-01 17:03:37,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2087: done (0 left) 2014-07-01 17:03:37,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: delete oid 240 current snap is 217 2014-07-01 17:03:37,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2088: done (0 left) 2014-07-01 17:03:37,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2089: snap_remove snap 189 2014-07-01 17:03:38,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:2089: done (0 left) 2014-07-01 17:03:38,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: rollback oid 401 current snap is 217 2014-07-01 17:03:38,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 401 to 173 2014-07-01 17:03:38,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: write oid 196 current snap is 217 2014-07-01 17:03:38,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: seq_num 1081 ranges {450037=1} 2014-07-01 17:03:38,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: writing mira0437250-196 from 450037 to 450038 tid 1 2014-07-01 17:03:39,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: write oid 229 current snap is 217 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: seq_num 1082 ranges {510195=716906,1710194=1} 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: writing mira0437250-229 from 510195 to 1227101 tid 1 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: writing mira0437250-229 from 1710194 to 1710195 tid 2 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: write oid 308 current snap is 217 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: seq_num 1083 ranges {606449=441159,1406448=1} 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0437250-308 from 606449 to 1047608 tid 1 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: writing mira0437250-308 from 1406448 to 1406449 tid 2 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: write oid 350 current snap is 217 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: seq_num 1084 ranges {550255=412963,1492420=539756,2150254=1} 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: writing mira0437250-350 from 550255 to 963218 tid 1 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: writing mira0437250-350 from 1492420 to 2032176 tid 2 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: writing mira0437250-350 from 2150254 to 2150255 tid 3 2014-07-01 17:03:39,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: snap_create 2014-07-01 17:03:39,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 1 to mira0437250-196 2014-07-01 17:03:39,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 987 (ObjNum 400 snap 0 seq_num 400) dirty exists 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: finishing write tid 1 to mira0437250-229 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: finishing write tid 2 to mira0437250-229 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: finishing write tid 4 to mira0437250-229 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 3 to mira0437250-196 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: finishing write tid 4 to mira0437250-196 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 196 v 903 (ObjNum 1081 snap 217 seq_num 1081) dirty exists 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: finishing write tid 5 to mira0437250-229 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 862 (ObjNum 1082 snap 217 seq_num 1082) dirty exists 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 1 to mira0437250-308 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 2 to mira0437250-308 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 4 to mira0437250-308 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: finishing write tid 5 to mira0437250-308 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 524 (ObjNum 1083 snap 217 seq_num 1083) dirty exists 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: finishing write tid 1 to mira0437250-350 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: finishing write tid 2 to mira0437250-350 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: finishing write tid 3 to mira0437250-350 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: finishing write tid 5 to mira0437250-350 2014-07-01 17:03:39,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: finishing write tid 6 to mira0437250-350 2014-07-01 17:03:39,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 920 (ObjNum 1084 snap 217 seq_num 1084) dirty exists 2014-07-01 17:03:39,556.556 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pgp_num 46' 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2090: done (5 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2091: done (4 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2092: done (3 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2093: done (2 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2094: done (1 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2095: done (0 left) 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: write oid 248 current snap is 218 2014-07-01 17:03:39,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: seq_num 1085 ranges {495084=419839,1338775=609952,2536782=358302} 2014-07-01 17:03:39,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: writing mira0437250-248 from 495084 to 914923 tid 1 2014-07-01 17:03:39,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: writing mira0437250-248 from 1338775 to 1948727 tid 2 2014-07-01 17:03:39,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: writing mira0437250-248 from 2536782 to 2895084 tid 3 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: snap_remove snap 173 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 1 to mira0437250-248 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 2 to mira0437250-248 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 3 to mira0437250-248 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 5 to mira0437250-248 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: finishing write tid 6 to mira0437250-248 2014-07-01 17:03:40,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 747 (ObjNum 1085 snap 218 seq_num 1085) dirty exists 2014-07-01 17:03:40,913.913 INFO:teuthology.task.rados.rados.0.mira043.stdout:2096: done (1 left) 2014-07-01 17:03:40,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2097: done (0 left) 2014-07-01 17:03:40,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: read oid 395 snap -1 2014-07-01 17:03:40,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2099: delete oid 420 current snap is 218 2014-07-01 17:03:40,930.930 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pgp_num to 46 2014-07-01 17:03:41,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2099: done (1 left) 2014-07-01 17:03:41,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: rollback oid 6 current snap is 218 2014-07-01 17:03:41,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 6 to 188 2014-07-01 17:03:41,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: write oid 313 current snap is 218 2014-07-01 17:03:41,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: seq_num 1087 ranges {650472=764472,2047296=501766,3172961=677511} 2014-07-01 17:03:41,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: writing mira0437250-313 from 650472 to 1414944 tid 1 2014-07-01 17:03:41,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: writing mira0437250-313 from 2047296 to 2549062 tid 2 2014-07-01 17:03:41,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: writing mira0437250-313 from 3172961 to 3850472 tid 3 2014-07-01 17:03:41,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: write oid 66 current snap is 218 2014-07-01 17:03:41,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: seq_num 1088 ranges {595301=1} 2014-07-01 17:03:41,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: writing mira0437250-66 from 595301 to 595302 tid 1 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2103: read oid 282 snap 158 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: read oid 7 snap -1 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: write oid 88 current snap is 218 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: seq_num 1089 ranges {290530=1} 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: writing mira0437250-88 from 290530 to 290531 tid 1 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: read oid 312 snap -1 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: write oid 398 current snap is 218 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: seq_num 1090 ranges {635361=400000} 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: writing mira0437250-398 from 635361 to 1035361 tid 1 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: snap_create 2014-07-01 17:03:41,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: expect (ObjNum 1006 snap 196 seq_num 1006) 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: finishing write tid 1 to mira0437250-313 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: finishing write tid 2 to mira0437250-313 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: finishing write tid 3 to mira0437250-313 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: finishing write tid 5 to mira0437250-313 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: finishing write tid 6 to mira0437250-313 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 870 (ObjNum 1087 snap 218 seq_num 1087) dirty exists 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 1 to mira0437250-398 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 3 to mira0437250-398 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: finishing write tid 4 to mira0437250-398 2014-07-01 17:03:41,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 560 (ObjNum 1090 snap 218 seq_num 1090) dirty exists 2014-07-01 17:03:41,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2101: done (9 left) 2014-07-01 17:03:41,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2103: done (8 left) 2014-07-01 17:03:41,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2106: done (7 left) 2014-07-01 17:03:41,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2107: done (6 left) 2014-07-01 17:03:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2108: done (5 left) 2014-07-01 17:03:41,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2109: snap_create 2014-07-01 17:03:42,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 0 (ObjNum 29 snap 0 seq_num 1) dirty dne 2014-07-01 17:03:42,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: expect (ObjNum 394 snap 0 seq_num 394) 2014-07-01 17:03:42,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: finishing write tid 1 to mira0437250-66 2014-07-01 17:03:42,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: finishing write tid 3 to mira0437250-66 2014-07-01 17:03:42,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: finishing write tid 4 to mira0437250-66 2014-07-01 17:03:42,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 906 (ObjNum 1088 snap 218 seq_num 1088) dirty exists 2014-07-01 17:03:42,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2098: done (5 left) 2014-07-01 17:03:42,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2100: done (4 left) 2014-07-01 17:03:42,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2102: done (3 left) 2014-07-01 17:03:42,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:2109: done (2 left) 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2110: rollback oid 251 current snap is 220 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 251 to 158 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: rollback oid 58 current snap is 220 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 58 to 207 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: write oid 95 current snap is 220 2014-07-01 17:03:42,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: seq_num 1091 ranges {696543=439210,1729809=417619,2296542=1} 2014-07-01 17:03:43,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: writing mira0437250-95 from 696543 to 1135753 tid 1 2014-07-01 17:03:43,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: writing mira0437250-95 from 1729809 to 2147428 tid 2 2014-07-01 17:03:43,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: writing mira0437250-95 from 2296542 to 2296543 tid 3 2014-07-01 17:03:43,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: read oid 401 snap -1 2014-07-01 17:03:43,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: write oid 129 current snap is 220 2014-07-01 17:03:43,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: seq_num 1092 ranges {791773=411014,1842917=148856} 2014-07-01 17:03:43,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: writing mira0437250-129 from 791773 to 1202787 tid 1 2014-07-01 17:03:43,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: writing mira0437250-129 from 1842917 to 1991773 tid 2 2014-07-01 17:03:43,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: write oid 430 current snap is 220 2014-07-01 17:03:43,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: seq_num 1093 ranges {736602=535266,1674424=708409,2736601=1} 2014-07-01 17:03:43,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0437250-430 from 736602 to 1271868 tid 1 2014-07-01 17:03:43,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0437250-430 from 1674424 to 2382833 tid 2 2014-07-01 17:03:43,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: writing mira0437250-430 from 2736601 to 2736602 tid 3 2014-07-01 17:03:43,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: write oid 383 current snap is 220 2014-07-01 17:03:43,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: seq_num 1094 ranges {679384=790719,2167309=778604,3479383=1} 2014-07-01 17:03:43,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: writing mira0437250-383 from 679384 to 1470103 tid 1 2014-07-01 17:03:43,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: writing mira0437250-383 from 2167309 to 2945913 tid 2 2014-07-01 17:03:43,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: writing mira0437250-383 from 3479383 to 3479384 tid 3 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: write oid 9 current snap is 220 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: seq_num 1095 ranges {775638=513947,2032866=600223,3175637=1} 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: writing mira0437250-9 from 775638 to 1289585 tid 1 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: writing mira0437250-9 from 2032866 to 2633089 tid 2 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: writing mira0437250-9 from 3175637 to 3175638 tid 3 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: write oid 65 current snap is 220 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: seq_num 1096 ranges {588244=485751,1579702=670419,2777270=610974} 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: writing mira0437250-65 from 588244 to 1073995 tid 1 2014-07-01 17:03:43,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: writing mira0437250-65 from 1579702 to 2250121 tid 2 2014-07-01 17:03:43,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: writing mira0437250-65 from 2777270 to 3388244 tid 3 2014-07-01 17:03:43,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: delete oid 62 current snap is 220 2014-07-01 17:03:43,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 651 (ObjNum 250 snap 0 seq_num 250) dirty exists 2014-07-01 17:03:43,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 1 to mira0437250-88 2014-07-01 17:03:43,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 3 to mira0437250-88 2014-07-01 17:03:43,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 1 to mira0437250-65 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 2 to mira0437250-65 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2110: done (11 left) 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2119: done (10 left) 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: write oid 115 current snap is 220 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: seq_num 1098 ranges {476879=400000} 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: writing mira0437250-115 from 476879 to 876879 tid 1 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2121: delete oid 82 current snap is 220 2014-07-01 17:03:43,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 3 to mira0437250-65 2014-07-01 17:03:43,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 5 to mira0437250-65 2014-07-01 17:03:43,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: finishing write tid 6 to mira0437250-65 2014-07-01 17:03:43,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 715 (ObjNum 1096 snap 220 seq_num 1096) dirty exists 2014-07-01 17:03:43,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: finishing write tid 1 to mira0437250-383 2014-07-01 17:03:43,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: finishing write tid 2 to mira0437250-383 2014-07-01 17:03:43,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: finishing write tid 3 to mira0437250-383 2014-07-01 17:03:43,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: finishing write tid 5 to mira0437250-383 2014-07-01 17:03:43,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: finishing write tid 1 to mira0435601-360 2014-07-01 17:03:43,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: finishing write tid 2 to mira0435601-360 2014-07-01 17:03:43,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: expect (ObjNum 1841 snap 0 seq_num 1841) 2014-07-01 17:03:44,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: expect (ObjNum 6 snap 0 seq_num 6) 2014-07-01 17:03:45,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: finishing write tid 3 to mira0435601-360 2014-07-01 17:03:45,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: finishing write tid 5 to mira0435601-360 2014-07-01 17:03:45,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: finishing write tid 6 to mira0435601-360 2014-07-01 17:03:45,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 192 (ObjNum 2014 snap 0 seq_num 2014) dirty exists 2014-07-01 17:03:45,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: finishing write tid 1 to mira0435601-435 2014-07-01 17:03:45,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: done (7 left) 2014-07-01 17:03:45,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: done (6 left) 2014-07-01 17:03:45,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: done (5 left) 2014-07-01 17:03:45,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: done (4 left) 2014-07-01 17:03:45,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: done (3 left) 2014-07-01 17:03:45,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3246: read oid 342 snap -1 2014-07-01 17:03:45,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: finishing write tid 3 to mira0435601-435 2014-07-01 17:03:45,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: finishing write tid 4 to mira0435601-435 2014-07-01 17:03:45,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 365 (ObjNum 2015 snap 0 seq_num 2015) dirty exists 2014-07-01 17:03:45,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:3245: expect (ObjNum 1929 snap 0 seq_num 1929) 2014-07-01 17:03:45,947.947 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:45,947.947 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:45,947.947 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2014-07-01 17:03:45,947.947 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 1' 2014-07-01 17:03:46,609.609 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 1 (8655362) 2014-07-01 17:03:46,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 1 to mira0437250-430 2014-07-01 17:03:46,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 2 to mira0437250-430 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 3 to mira0437250-430 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: finishing write tid 4 to mira0437250-88 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 923 (ObjNum 1089 snap 218 seq_num 1089) dirty exists 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 5 to mira0437250-430 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: finishing write tid 6 to mira0437250-430 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 782 (ObjNum 1093 snap 220 seq_num 1093) dirty exists 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: finishing write tid 6 to mira0437250-383 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 920 (ObjNum 1094 snap 220 seq_num 1094) dirty exists 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: finishing write tid 1 to mira0437250-95 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: finishing write tid 2 to mira0437250-95 2014-07-01 17:03:46,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: expect (ObjNum 400 snap 0 seq_num 400) 2014-07-01 17:03:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: finishing write tid 1 to mira0435601-434 2014-07-01 17:03:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: finishing write tid 3 to mira0435601-434 2014-07-01 17:03:48,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: finishing write tid 4 to mira0435601-434 2014-07-01 17:03:48,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 240 (ObjNum 2016 snap 0 seq_num 2016) dirty exists 2014-07-01 17:03:48,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: done (3 left) 2014-07-01 17:03:48,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: done (2 left) 2014-07-01 17:03:48,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3245: done (1 left) 2014-07-01 17:03:48,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3246: done (0 left) 2014-07-01 17:03:48,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:3247: read oid 379 snap -1 2014-07-01 17:03:48,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 82 current snap is 0 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: seq_num 2017 ranges {0=696320,696320=401408} 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: writing mira0435601-82 from 0 to 696320 tid 1 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: writing mira0435601-82 from 696320 to 1097728 tid 2 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 93 current snap is 0 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: seq_num 2018 ranges {2039808=724992,2764800=778240,3543040=745472,4288512=81920} 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: writing mira0435601-93 from 2039808 to 2764800 tid 1 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: finishing write tid 3 to mira0437250-95 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: finishing write tid 5 to mira0437250-95 2014-07-01 17:03:48,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: finishing write tid 6 to mira0437250-95 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 633 (ObjNum 1091 snap 220 seq_num 1091) dirty exists 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: finishing write tid 1 to mira0437250-115 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: finishing write tid 3 to mira0437250-115 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2104: done (11 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2105: done (10 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2112: done (9 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2113: done (8 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2115: done (7 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2116: done (6 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2118: done (5 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2121: done (4 left) 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: snap_create 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: finishing write tid 1 to mira0437250-9 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: finishing write tid 2 to mira0437250-9 2014-07-01 17:03:48,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: finishing write tid 3 to mira0437250-9 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: finishing write tid 5 to mira0437250-9 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: finishing write tid 4 to mira0437250-115 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 745 (ObjNum 1098 snap 220 seq_num 1098) dirty exists 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: finishing write tid 6 to mira0437250-9 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 9 v 1161 (ObjNum 1095 snap 220 seq_num 1095) dirty exists 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 922 (ObjNum 57 snap 0 seq_num 57) dirty exists 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: writing mira0435601-93 from 2764800 to 3543040 tid 2 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: writing mira0435601-93 from 3543040 to 4288512 tid 3 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: writing mira0435601-93 from 4288512 to 4370432 tid 4 2014-07-01 17:03:48,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3250: read oid 158 snap -1 2014-07-01 17:03:48,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3247: expect (ObjNum 1724 snap 0 seq_num 1724) 2014-07-01 17:03:48,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2111: done (4 left) 2014-07-01 17:03:48,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2117: done (3 left) 2014-07-01 17:03:48,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2120: done (2 left) 2014-07-01 17:03:48,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2122: done (1 left) 2014-07-01 17:03:48,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: delete oid 303 current snap is 221 2014-07-01 17:03:50,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 1 to mira0437250-129 2014-07-01 17:03:51,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: finishing write tid 1 to mira0435601-82 2014-07-01 17:03:51,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: finishing write tid 2 to mira0435601-82 2014-07-01 17:03:51,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: finishing write tid 4 to mira0435601-82 2014-07-01 17:03:51,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: finishing write tid 5 to mira0435601-82 2014-07-01 17:03:51,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 375 (ObjNum 2017 snap 0 seq_num 2017) dirty exists 2014-07-01 17:03:51,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3250: expect (ObjNum 1980 snap 0 seq_num 1980) 2014-07-01 17:03:51,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 2 to mira0437250-129 2014-07-01 17:03:51,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 4 to mira0437250-129 2014-07-01 17:03:51,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: finishing write tid 5 to mira0437250-129 2014-07-01 17:03:51,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 1236 (ObjNum 1092 snap 220 seq_num 1092) dirty exists 2014-07-01 17:03:51,619.619 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:03:51,619.619 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:03:51,619.619 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:03:51,619.619 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:03:51,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2114: done (1 left) 2014-07-01 17:03:51,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:2123: done (0 left) 2014-07-01 17:03:51,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: write oid 270 current snap is 221 2014-07-01 17:03:51,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: seq_num 1101 ranges {578120=533317,1776346=656467,2578119=1} 2014-07-01 17:03:51,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: writing mira0437250-270 from 578120 to 1111437 tid 1 2014-07-01 17:03:51,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: writing mira0437250-270 from 1776346 to 2432813 tid 2 2014-07-01 17:03:51,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: writing mira0437250-270 from 2578119 to 2578120 tid 3 2014-07-01 17:03:51,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2125: read oid 320 snap -1 2014-07-01 17:03:51,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: write oid 248 current snap is 221 2014-07-01 17:03:51,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: seq_num 1102 ranges {520902=788769,2020654=478086,3145198=175704} 2014-07-01 17:03:51,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: writing mira0437250-248 from 520902 to 1309671 tid 1 2014-07-01 17:03:51,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: writing mira0437250-248 from 2020654 to 2498740 tid 2 2014-07-01 17:03:51,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: writing mira0437250-248 from 3145198 to 3320902 tid 3 2014-07-01 17:03:51,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2127: snap_remove snap 158 2014-07-01 17:03:51,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2125: expect (ObjNum 319 snap 0 seq_num 319) 2014-07-01 17:03:52,181.181 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: finishing write tid 1 to mira0437250-270 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: finishing write tid 2 to mira0437250-270 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: finishing write tid 3 to mira0437250-270 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: finishing write tid 5 to mira0437250-270 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: finishing write tid 6 to mira0437250-270 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 270 v 1165 (ObjNum 1101 snap 221 seq_num 1101) dirty exists 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2124: done (3 left) 2014-07-01 17:03:52,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2125: done (2 left) 2014-07-01 17:03:52,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:2127: done (1 left) 2014-07-01 17:03:52,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: write oid 357 current snap is 221 2014-07-01 17:03:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: seq_num 1103 ranges {65731=1} 2014-07-01 17:03:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: writing mira0437250-357 from 65731 to 65732 tid 1 2014-07-01 17:03:52,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: snap_create 2014-07-01 17:03:53,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: finishing write tid 1 to mira0437250-248 2014-07-01 17:03:53,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: finishing write tid 2 to mira0437250-248 2014-07-01 17:03:53,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: finishing write tid 1 to mira0437250-357 2014-07-01 17:03:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: finishing write tid 3 to mira0437250-357 2014-07-01 17:03:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: finishing write tid 3 to mira0437250-248 2014-07-01 17:03:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: finishing write tid 5 to mira0437250-248 2014-07-01 17:03:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: finishing write tid 6 to mira0437250-248 2014-07-01 17:03:53,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 752 (ObjNum 1102 snap 221 seq_num 1102) dirty exists 2014-07-01 17:03:53,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: finishing write tid 4 to mira0437250-357 2014-07-01 17:03:53,274.274 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 1164 (ObjNum 1103 snap 221 seq_num 1103) dirty exists 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 1 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 2 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 3 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 4 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 6 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: finishing write tid 7 to mira0435601-93 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 130 (ObjNum 2018 snap 0 seq_num 2018) dirty exists 2014-07-01 17:03:53,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3247: done (3 left) 2014-07-01 17:03:53,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: done (2 left) 2014-07-01 17:03:53,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: done (1 left) 2014-07-01 17:03:53,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3250: done (0 left) 2014-07-01 17:03:53,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3251: read oid 223 snap -1 2014-07-01 17:03:53,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3252: delete oid 290 current snap is 0 2014-07-01 17:03:53,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2126: done (2 left) 2014-07-01 17:03:53,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2128: done (1 left) 2014-07-01 17:03:53,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2129: done (0 left) 2014-07-01 17:03:53,437.437 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: snap_remove snap 216 2014-07-01 17:03:53,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3251: expect (ObjNum 1984 snap 0 seq_num 1984) 2014-07-01 17:03:54,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:3251: done (1 left) 2014-07-01 17:03:54,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:3252: done (0 left) 2014-07-01 17:03:54,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: read oid 94 snap -1 2014-07-01 17:03:54,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:3254: delete oid 308 current snap is 0 2014-07-01 17:03:54,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: expect (ObjNum 1377 snap 0 seq_num 1377) 2014-07-01 17:03:55,575.575 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:03:56,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2130: done (0 left) 2014-07-01 17:03:56,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: read oid 468 snap -1 2014-07-01 17:03:56,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: write oid 158 current snap is 222 2014-07-01 17:03:56,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: seq_num 1104 ranges {560961=484826,1813846=618476,2845824=601117,3760960=1} 2014-07-01 17:03:56,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: writing mira0437250-158 from 560961 to 1045787 tid 1 2014-07-01 17:03:56,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: writing mira0437250-158 from 1813846 to 2432322 tid 2 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: writing mira0437250-158 from 2845824 to 3446941 tid 3 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: writing mira0437250-158 from 3760960 to 3760961 tid 4 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: write oid 478 current snap is 222 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: seq_num 1105 ranges {622143=400000} 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: writing mira0437250-478 from 622143 to 1022143 tid 1 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2134: rollback oid 110 current snap is 222 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 110 to 218 2014-07-01 17:03:56,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: expect (ObjNum 467 snap 0 seq_num 467) 2014-07-01 17:03:56,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2131: done (3 left) 2014-07-01 17:03:56,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: write oid 273 current snap is 222 2014-07-01 17:03:56,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: seq_num 1106 ranges {566973=463507,1454591=312382} 2014-07-01 17:03:56,330.330 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: writing mira0437250-273 from 566973 to 1030480 tid 1 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: writing mira0437250-273 from 1454591 to 1766973 tid 2 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: read oid 497 snap -1 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: delete oid 34 current snap is 222 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 1 to mira0437250-158 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2136: done (5 left) 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2137: done (4 left) 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: write oid 219 current snap is 222 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: seq_num 1108 ranges {607032=409163,1648806=558226} 2014-07-01 17:03:56,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: writing mira0437250-219 from 607032 to 1016195 tid 1 2014-07-01 17:03:56,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: writing mira0437250-219 from 1648806 to 2207032 tid 2 2014-07-01 17:03:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: snap_create 2014-07-01 17:03:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 2 to mira0437250-158 2014-07-01 17:03:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 3 to mira0437250-158 2014-07-01 17:03:56,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 4 to mira0437250-158 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 6 to mira0437250-158 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: finishing write tid 7 to mira0437250-158 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 158 v 739 (ObjNum 1104 snap 222 seq_num 1104) dirty exists 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 1 to mira0437250-478 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 3 to mira0437250-478 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 893 (ObjNum 109 snap 0 seq_num 109) dirty exists 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: finishing write tid 4 to mira0437250-478 2014-07-01 17:03:56,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 478 v 926 (ObjNum 1105 snap 222 seq_num 1105) dirty exists 2014-07-01 17:03:56,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: finishing write tid 1 to mira0437250-273 2014-07-01 17:03:56,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: finishing write tid 2 to mira0437250-273 2014-07-01 17:03:56,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: finishing write tid 4 to mira0437250-273 2014-07-01 17:03:56,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 1 to mira0437250-219 2014-07-01 17:03:56,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 2 to mira0437250-219 2014-07-01 17:03:56,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: finishing write tid 5 to mira0437250-273 2014-07-01 17:03:56,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 999 (ObjNum 1106 snap 222 seq_num 1106) dirty exists 2014-07-01 17:03:56,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 4 to mira0437250-219 2014-07-01 17:03:56,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: finishing write tid 5 to mira0437250-219 2014-07-01 17:03:56,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 866 (ObjNum 1108 snap 222 seq_num 1108) dirty exists 2014-07-01 17:03:57,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:2132: done (5 left) 2014-07-01 17:03:57,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:2133: done (4 left) 2014-07-01 17:03:57,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2134: done (3 left) 2014-07-01 17:03:57,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2135: done (2 left) 2014-07-01 17:03:57,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2138: done (1 left) 2014-07-01 17:03:57,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2139: done (0 left) 2014-07-01 17:03:57,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:2140: delete oid 430 current snap is 223 2014-07-01 17:03:57,306.306 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:03:57,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:2140: done (0 left) 2014-07-01 17:03:57,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: snap_remove snap 220 2014-07-01 17:03:58,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:2141: done (0 left) 2014-07-01 17:03:58,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2142: rollback oid 6 current snap is 223 2014-07-01 17:03:58,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 6 to 211 2014-07-01 17:03:58,294.294 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: delete oid 380 current snap is 223 2014-07-01 17:03:58,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 0 (ObjNum 16144144 snap 0 seq_num 0) dirty dne 2014-07-01 17:03:58,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2142: done (1 left) 2014-07-01 17:03:58,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2143: done (0 left) 2014-07-01 17:03:58,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2144: snap_remove snap 195 2014-07-01 17:03:59,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2144: done (0 left) 2014-07-01 17:03:59,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: rollback oid 78 current snap is 223 2014-07-01 17:03:59,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 78 to 218 2014-07-01 17:03:59,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: write oid 475 current snap is 223 2014-07-01 17:03:59,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: seq_num 1111 ranges {707250=512096,1955107=646738,3152577=680459,3907249=1} 2014-07-01 17:03:59,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: writing mira0437250-475 from 707250 to 1219346 tid 1 2014-07-01 17:03:59,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: writing mira0437250-475 from 1955107 to 2601845 tid 2 2014-07-01 17:03:59,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: writing mira0437250-475 from 3152577 to 3833036 tid 3 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: writing mira0437250-475 from 3907249 to 3907250 tid 4 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2147: snap_remove snap 203 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 0 (ObjNum 15639584 snap 0 seq_num 15636224) dirty dne 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 1 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 2 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 3 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 4 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 6 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: finishing write tid 7 to mira0437250-475 2014-07-01 17:03:59,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 1083 (ObjNum 1111 snap 223 seq_num 1111) dirty exists 2014-07-01 17:04:00,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2145: done (2 left) 2014-07-01 17:04:00,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2146: done (1 left) 2014-07-01 17:04:00,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2147: done (0 left) 2014-07-01 17:04:00,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: read oid 419 snap -1 2014-07-01 17:04:00,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2149: read oid 390 snap -1 2014-07-01 17:04:00,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: snap_create 2014-07-01 17:04:00,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: expect (ObjNum 418 snap 0 seq_num 418) 2014-07-01 17:04:00,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: done (1 left) 2014-07-01 17:04:00,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3254: done (0 left) 2014-07-01 17:04:00,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 66 current snap is 0 2014-07-01 17:04:00,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: seq_num 2021 ranges {2875392=544768,3420160=774144,4194304=430080} 2014-07-01 17:04:00,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: writing mira0435601-66 from 2875392 to 3420160 tid 1 2014-07-01 17:04:00,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: writing mira0435601-66 from 3420160 to 4194304 tid 2 2014-07-01 17:04:00,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: writing mira0435601-66 from 4194304 to 4624384 tid 3 2014-07-01 17:04:00,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 208 current snap is 0 2014-07-01 17:04:00,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: seq_num 2022 ranges {3375104=569344} 2014-07-01 17:04:00,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: writing mira0435601-208 from 3375104 to 3944448 tid 1 2014-07-01 17:04:00,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 17:04:00,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: seq_num 2023 ranges {8642560=442368} 2014-07-01 17:04:00,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: writing mira0435601-94 from 8642560 to 9084928 tid 1 2014-07-01 17:04:00,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 65 current snap is 0 2014-07-01 17:04:00,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: seq_num 2024 ranges {2883584=315392} 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: writing mira0435601-65 from 2883584 to 3198976 tid 1 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3259: delete oid 382 current snap is 0 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: finishing write tid 1 to mira0435601-66 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: finishing write tid 2 to mira0435601-66 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: finishing write tid 3 to mira0435601-66 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: finishing write tid 5 to mira0435601-66 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: finishing write tid 6 to mira0435601-66 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 236 (ObjNum 2021 snap 0 seq_num 2021) dirty exists 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: finishing write tid 1 to mira0435601-208 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: finishing write tid 3 to mira0435601-208 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: finishing write tid 4 to mira0435601-208 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 208 v 367 (ObjNum 2022 snap 0 seq_num 2022) dirty exists 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: finishing write tid 1 to mira0435601-94 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: finishing write tid 3 to mira0435601-94 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: finishing write tid 4 to mira0435601-94 2014-07-01 17:04:00,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 255 (ObjNum 2023 snap 0 seq_num 2023) dirty exists 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: done (4 left) 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: done (3 left) 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: done (2 left) 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3259: done (1 left) 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 17:04:00,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: seq_num 2026 ranges {7639040=663552,8302592=741376,9043968=483328,9527296=380928} 2014-07-01 17:04:00,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: writing mira0435601-27 from 7639040 to 8302592 tid 1 2014-07-01 17:04:00,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: writing mira0435601-27 from 8302592 to 9043968 tid 2 2014-07-01 17:04:00,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: writing mira0435601-27 from 9043968 to 9527296 tid 3 2014-07-01 17:04:01,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: writing mira0435601-27 from 9527296 to 9908224 tid 4 2014-07-01 17:04:01,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 348 current snap is 0 2014-07-01 17:04:01,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: seq_num 2027 ranges {2682880=536576,3219456=413696,3633152=708608,4341760=475136,4816896=8192} 2014-07-01 17:04:01,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0435601-348 from 2682880 to 3219456 tid 1 2014-07-01 17:04:01,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0435601-348 from 3219456 to 3633152 tid 2 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0435601-348 from 3633152 to 4341760 tid 3 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0435601-348 from 4341760 to 4816896 tid 4 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0435601-348 from 4816896 to 4825088 tid 5 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: seq_num 2028 ranges {9494528=405504,9900032=798720,10698752=430080,11128832=376832} 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: writing mira0435601-313 from 9494528 to 9900032 tid 1 2014-07-01 17:04:01,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: writing mira0435601-313 from 9900032 to 10698752 tid 2 2014-07-01 17:04:01,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: writing mira0435601-313 from 10698752 to 11128832 tid 3 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: writing mira0435601-313 from 11128832 to 11505664 tid 4 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3263: read oid 312 snap -1 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: finishing write tid 1 to mira0435601-65 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: finishing write tid 3 to mira0435601-65 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: finishing write tid 4 to mira0435601-65 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 65 v 194 (ObjNum 2024 snap 0 seq_num 2024) dirty exists 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 1 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 2 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 3 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 4 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 6 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: finishing write tid 7 to mira0435601-27 2014-07-01 17:04:01,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 236 (ObjNum 2026 snap 0 seq_num 2026) dirty exists 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 1 to mira0435601-348 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 2 to mira0435601-348 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 3 to mira0435601-348 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 4 to mira0435601-348 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 5 to mira0435601-348 2014-07-01 17:04:01,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 7 to mira0435601-348 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 8 to mira0435601-348 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 318 (ObjNum 2027 snap 0 seq_num 2027) dirty exists 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 1 to mira0435601-313 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: done (4 left) 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: done (3 left) 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: done (2 left) 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 218 current snap is 0 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: seq_num 2029 ranges {0=483328,483328=581632,1064960=622592} 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: writing mira0435601-218 from 0 to 483328 tid 1 2014-07-01 17:04:01,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: writing mira0435601-218 from 483328 to 1064960 tid 2 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: writing mira0435601-218 from 1064960 to 1687552 tid 3 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 180 current snap is 0 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: seq_num 2030 ranges {2265088=757760,3022848=565248,3588096=237568} 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: writing mira0435601-180 from 2265088 to 3022848 tid 1 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: writing mira0435601-180 from 3022848 to 3588096 tid 2 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: writing mira0435601-180 from 3588096 to 3825664 tid 3 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3266: read oid 68 snap -1 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 2 to mira0435601-313 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 3 to mira0435601-313 2014-07-01 17:04:01,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 4 to mira0435601-313 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3267: read oid 411 snap -1 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 6 to mira0435601-313 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3263: expect (ObjNum 1712 snap 0 seq_num 1712) 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2148: done (2 left) 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2149: done (1 left) 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2150: done (0 left) 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: read oid 98 snap -1 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: write oid 13 current snap is 224 2014-07-01 17:04:01,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: seq_num 1112 ranges {402479=483901,1516792=715910,2800332=414802,3602478=1} 2014-07-01 17:04:01,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0437250-13 from 402479 to 886380 tid 1 2014-07-01 17:04:01,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0437250-13 from 1516792 to 2232702 tid 2 2014-07-01 17:04:01,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0437250-13 from 2800332 to 3215134 tid 3 2014-07-01 17:04:01,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: writing mira0437250-13 from 3602478 to 3602479 tid 4 2014-07-01 17:04:01,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2153: delete oid 332 current snap is 224 2014-07-01 17:04:01,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 17:04:01,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: finishing write tid 1 to mira0435601-218 2014-07-01 17:04:01,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: finishing write tid 7 to mira0435601-313 2014-07-01 17:04:01,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 224 (ObjNum 2028 snap 0 seq_num 2028) dirty exists 2014-07-01 17:04:01,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: finishing write tid 2 to mira0435601-218 2014-07-01 17:04:01,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: finishing write tid 3 to mira0435601-218 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: finishing write tid 5 to mira0435601-218 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: done (5 left) 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3263: done (4 left) 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3268: read oid 419 snap -1 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: finishing write tid 6 to mira0435601-218 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 266 (ObjNum 2029 snap 0 seq_num 2029) dirty exists 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: finishing write tid 1 to mira0435601-180 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: done (4 left) 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 237 current snap is 0 2014-07-01 17:04:01,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: seq_num 2031 ranges {2453504=626688,3080192=794624,3874816=8192} 2014-07-01 17:04:01,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: writing mira0435601-237 from 2453504 to 3080192 tid 1 2014-07-01 17:04:01,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: writing mira0435601-237 from 3080192 to 3874816 tid 2 2014-07-01 17:04:01,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: writing mira0435601-237 from 3874816 to 3883008 tid 3 2014-07-01 17:04:01,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 17:04:01,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: seq_num 2032 ranges {6651904=253952} 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: writing mira0435601-492 from 6651904 to 6905856 tid 1 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3271: read oid 80 snap -1 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 437 current snap is 0 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: seq_num 2033 ranges {73728=684032,757760=561152,1318912=241664} 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: writing mira0435601-437 from 73728 to 757760 tid 1 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: writing mira0435601-437 from 757760 to 1318912 tid 2 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: writing mira0435601-437 from 1318912 to 1560576 tid 3 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3273: delete oid 370 current snap is 0 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: finishing write tid 2 to mira0435601-180 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: finishing write tid 3 to mira0435601-180 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: finishing write tid 5 to mira0435601-180 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: finishing write tid 6 to mira0435601-180 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 304 (ObjNum 2030 snap 0 seq_num 2030) dirty exists 2014-07-01 17:04:01,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3266: expect (ObjNum 1706 snap 0 seq_num 1706) 2014-07-01 17:04:02,322.322 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:02,322.322 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:02,323.323 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:04:02,323.323 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:04:02,323.323 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:04:02,323.323 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 1 to mira0437250-13 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 2 to mira0437250-13 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 3 to mira0437250-13 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 4 to mira0437250-13 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 6 to mira0437250-13 2014-07-01 17:04:03,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: finishing write tid 7 to mira0437250-13 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 684 (ObjNum 1112 snap 224 seq_num 1112) dirty exists 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2151: done (2 left) 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2152: done (1 left) 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2153: done (0 left) 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2154: rollback oid 43 current snap is 224 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 43 to 199 2014-07-01 17:04:03,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:2155: rollback oid 108 current snap is 224 2014-07-01 17:04:03,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 108 to 188 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: write oid 388 current snap is 224 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: seq_num 1114 ranges {408491=462581,1308960=299531} 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: writing mira0437250-388 from 408491 to 871072 tid 1 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: writing mira0437250-388 from 1308960 to 1608491 tid 2 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2157: delete oid 279 current snap is 224 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 1225 (ObjNum 500 snap 1 seq_num 500) dirty exists 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 979 (ObjNum 107 snap 0 seq_num 107) dirty exists 2014-07-01 17:04:03,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 1 to mira0437250-388 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2154: done (3 left) 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2155: done (2 left) 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2157: done (1 left) 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2158: delete oid 336 current snap is 224 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2158: done (1 left) 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: read oid 263 snap -1 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: snap_remove snap 214 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 2 to mira0437250-388 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 4 to mira0437250-388 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: finishing write tid 5 to mira0437250-388 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 532 (ObjNum 1114 snap 224 seq_num 1114) dirty exists 2014-07-01 17:04:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: expect (ObjNum 1074 snap 214 seq_num 1074) 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2156: done (2 left) 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2159: done (1 left) 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2160: done (0 left) 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: read oid 350 snap -1 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: write oid 375 current snap is 224 2014-07-01 17:04:03,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: seq_num 1117 ranges {792356=681866,2015260=604806,2792355=1} 2014-07-01 17:04:03,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0437250-375 from 792356 to 1474222 tid 1 2014-07-01 17:04:03,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0437250-375 from 2015260 to 2620066 tid 2 2014-07-01 17:04:03,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: writing mira0437250-375 from 2792355 to 2792356 tid 3 2014-07-01 17:04:03,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2163: snap_create 2014-07-01 17:04:03,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: expect (ObjNum 1084 snap 217 seq_num 1084) 2014-07-01 17:04:04,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:3267: expect (ObjNum 1322 snap 0 seq_num 1322) 2014-07-01 17:04:04,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 1 to mira0437250-375 2014-07-01 17:04:04,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 2 to mira0437250-375 2014-07-01 17:04:04,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 3 to mira0437250-375 2014-07-01 17:04:04,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 5 to mira0437250-375 2014-07-01 17:04:04,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: finishing write tid 6 to mira0437250-375 2014-07-01 17:04:04,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 375 v 988 (ObjNum 1117 snap 224 seq_num 1117) dirty exists 2014-07-01 17:04:04,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2161: done (2 left) 2014-07-01 17:04:04,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2162: done (1 left) 2014-07-01 17:04:04,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2163: done (0 left) 2014-07-01 17:04:04,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:2164: rollback oid 311 current snap is 225 2014-07-01 17:04:04,806.806 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 311 to 219 2014-07-01 17:04:04,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: rollback oid 198 current snap is 225 2014-07-01 17:04:04,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 198 to 184 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: write oid 486 current snap is 225 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: seq_num 1118 ranges {603938=538343,1845745=523577,3003937=1} 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: writing mira0437250-486 from 603938 to 1142281 tid 1 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: writing mira0437250-486 from 1845745 to 2369322 tid 2 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: writing mira0437250-486 from 3003937 to 3003938 tid 3 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: read oid 440 snap -1 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 755 (ObjNum 938 snap 171 seq_num 938) dirty exists 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 1235 (ObjNum 857 snap 143 seq_num 857) dirty exists 2014-07-01 17:04:04,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 1 to mira0437250-486 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2164: done (3 left) 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2165: done (2 left) 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: read oid 114 snap 156 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2169: read oid 473 snap -1 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: snap_create 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 2 to mira0437250-486 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 3 to mira0437250-486 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 5 to mira0437250-486 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: finishing write tid 6 to mira0437250-486 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 762 (ObjNum 1118 snap 225 seq_num 1118) dirty exists 2014-07-01 17:04:04,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: expect (ObjNum 695 snap 89 seq_num 695) 2014-07-01 17:04:05,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: expect (ObjNum 113 snap 0 seq_num 113) 2014-07-01 17:04:06,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:2166: done (4 left) 2014-07-01 17:04:06,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2167: done (3 left) 2014-07-01 17:04:06,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2168: done (2 left) 2014-07-01 17:04:06,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2169: done (1 left) 2014-07-01 17:04:06,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:2170: done (0 left) 2014-07-01 17:04:06,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: write oid 359 current snap is 226 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: seq_num 1119 ranges {700192=510147,1657029=745196,2700191=1} 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: writing mira0437250-359 from 700192 to 1210339 tid 1 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: writing mira0437250-359 from 1657029 to 2402225 tid 2 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: writing mira0437250-359 from 2700191 to 2700192 tid 3 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2172: delete oid 316 current snap is 226 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 1 to mira0437250-359 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2172: done (1 left) 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: snap_remove snap 207 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 2 to mira0437250-359 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 3 to mira0437250-359 2014-07-01 17:04:06,309.309 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 5 to mira0437250-359 2014-07-01 17:04:06,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: finishing write tid 6 to mira0437250-359 2014-07-01 17:04:06,310.310 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 857 (ObjNum 1119 snap 226 seq_num 1119) dirty exists 2014-07-01 17:04:06,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2171: done (1 left) 2014-07-01 17:04:06,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2173: done (0 left) 2014-07-01 17:04:06,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: delete oid 91 current snap is 226 2014-07-01 17:04:06,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2174: done (0 left) 2014-07-01 17:04:06,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: delete oid 128 current snap is 226 2014-07-01 17:04:06,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2175: done (0 left) 2014-07-01 17:04:06,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2176: delete oid 69 current snap is 226 2014-07-01 17:04:06,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:2176: done (0 left) 2014-07-01 17:04:06,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: snap_create 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: finishing write tid 1 to mira0435601-237 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: finishing write tid 2 to mira0435601-237 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: finishing write tid 3 to mira0435601-237 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: finishing write tid 5 to mira0435601-237 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: finishing write tid 6 to mira0435601-237 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 134 (ObjNum 2031 snap 0 seq_num 2031) dirty exists 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: finishing write tid 1 to mira0435601-492 2014-07-01 17:04:06,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: done (8 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3266: done (7 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3267: done (6 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3268: done (5 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: done (4 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3273: done (3 left) 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 178 current snap is 0 2014-07-01 17:04:06,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: seq_num 2035 ranges {4157440=475136,4632576=774144,5406720=602112,6008832=229376} 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: writing mira0435601-178 from 4157440 to 4632576 tid 1 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: writing mira0435601-178 from 4632576 to 5406720 tid 2 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: writing mira0435601-178 from 5406720 to 6008832 tid 3 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: writing mira0435601-178 from 6008832 to 6238208 tid 4 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3275: read oid 127 snap -1 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 4 current snap is 0 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: seq_num 2036 ranges {3002368=503808,3506176=401408} 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: writing mira0435601-4 from 3002368 to 3506176 tid 1 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: writing mira0435601-4 from 3506176 to 3907584 tid 2 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 165 current snap is 0 2014-07-01 17:04:07,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: seq_num 2037 ranges {0=618496,618496=434176,1052672=704512,1757184=65536} 2014-07-01 17:04:07,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: writing mira0435601-165 from 0 to 618496 tid 1 2014-07-01 17:04:07,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: writing mira0435601-165 from 618496 to 1052672 tid 2 2014-07-01 17:04:07,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: writing mira0435601-165 from 1052672 to 1757184 tid 3 2014-07-01 17:04:07,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: writing mira0435601-165 from 1757184 to 1822720 tid 4 2014-07-01 17:04:07,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:3278: read oid 364 snap -1 2014-07-01 17:04:07,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: finishing write tid 3 to mira0435601-492 2014-07-01 17:04:07,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: finishing write tid 4 to mira0435601-492 2014-07-01 17:04:07,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 208 (ObjNum 2032 snap 0 seq_num 2032) dirty exists 2014-07-01 17:04:07,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: done (7 left) 2014-07-01 17:04:07,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 17:04:07,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: seq_num 2038 ranges {8531968=696320,9228288=770048,9998336=32768} 2014-07-01 17:04:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: writing mira0435601-409 from 8531968 to 9228288 tid 1 2014-07-01 17:04:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: writing mira0435601-409 from 9228288 to 9998336 tid 2 2014-07-01 17:04:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: writing mira0435601-409 from 9998336 to 10031104 tid 3 2014-07-01 17:04:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 17:04:07,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: seq_num 2039 ranges {9084928=319488} 2014-07-01 17:04:07,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: writing mira0435601-94 from 9084928 to 9404416 tid 1 2014-07-01 17:04:07,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 257 current snap is 0 2014-07-01 17:04:07,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: seq_num 2040 ranges {6590464=438272,7028736=585728,7614464=217088} 2014-07-01 17:04:07,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0435601-257 from 6590464 to 7028736 tid 1 2014-07-01 17:04:07,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0435601-257 from 7028736 to 7614464 tid 2 2014-07-01 17:04:07,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0435601-257 from 7614464 to 7831552 tid 3 2014-07-01 17:04:07,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 483 current snap is 0 2014-07-01 17:04:07,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: seq_num 2041 ranges {5632000=65536} 2014-07-01 17:04:07,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: writing mira0435601-483 from 5632000 to 5697536 tid 1 2014-07-01 17:04:07,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: read oid 434 snap -1 2014-07-01 17:04:07,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3271: expect (ObjNum 1052 snap 0 seq_num 1052) 2014-07-01 17:04:07,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2177: done (0 left) 2014-07-01 17:04:07,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: snap_remove snap 184 2014-07-01 17:04:07,792.792 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:07,792.792 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:07,792.792 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_2 2014-07-01 17:04:07,792.792 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:04:08,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:2178: done (0 left) 2014-07-01 17:04:08,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:2179: snap_create 2014-07-01 17:04:08,555.555 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:04:09,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2179: done (0 left) 2014-07-01 17:04:09,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: read oid 238 snap -1 2014-07-01 17:04:09,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: write oid 7 current snap is 228 2014-07-01 17:04:09,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: seq_num 1124 ranges {690068=556689,1890067=1} 2014-07-01 17:04:09,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: writing mira0437250-7 from 690068 to 1246757 tid 1 2014-07-01 17:04:09,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: writing mira0437250-7 from 1890067 to 1890068 tid 2 2014-07-01 17:04:09,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:2182: rollback oid 288 current snap is 228 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 288 to 188 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2180: done (2 left) 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: write oid 72 current snap is 228 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: seq_num 1125 ranges {785298=680941,1585297=1} 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 1 to mira0435601-437 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 2 to mira0435601-437 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 3 to mira0435601-437 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 5 to mira0435601-437 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 6 to mira0435601-437 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 269 (ObjNum 2033 snap 0 seq_num 2033) dirty exists 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 1 to mira0435601-178 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 2 to mira0435601-178 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 3 to mira0435601-178 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 4 to mira0435601-178 2014-07-01 17:04:09,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 6 to mira0435601-178 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: finishing write tid 7 to mira0435601-178 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 363 (ObjNum 2035 snap 0 seq_num 2035) dirty exists 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: finishing write tid 1 to mira0435601-4 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3275: expect (ObjNum 1766 snap 0 seq_num 1766) 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: writing mira0437250-72 from 785298 to 1466239 tid 1 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: writing mira0437250-72 from 1585297 to 1585298 tid 2 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: read oid 477 snap -1 2014-07-01 17:04:09,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: finishing write tid 1 to mira0437250-7 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: finishing write tid 2 to mira0437250-7 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: finishing write tid 4 to mira0437250-7 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: finishing write tid 5 to mira0437250-7 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 7 v 1220 (ObjNum 1124 snap 228 seq_num 1124) dirty exists 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 798 (ObjNum 755 snap 110 seq_num 755) dirty exists 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2181: done (3 left) 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2182: done (2 left) 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2185: read oid 203 snap -1 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: read oid 489 snap -1 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2187: snap_remove snap 226 2014-07-01 17:04:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: expect (ObjNum 680 snap 79 seq_num 680) 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 1 to mira0437250-72 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 2 to mira0437250-72 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 4 to mira0437250-72 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: finishing write tid 5 to mira0437250-72 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 1434 (ObjNum 1125 snap 228 seq_num 1125) dirty exists 2014-07-01 17:04:09,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: expect (ObjNum 803 snap 129 seq_num 803) 2014-07-01 17:04:10,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2183: done (4 left) 2014-07-01 17:04:10,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2184: done (3 left) 2014-07-01 17:04:10,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2185: done (2 left) 2014-07-01 17:04:10,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2186: done (1 left) 2014-07-01 17:04:10,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2187: done (0 left) 2014-07-01 17:04:10,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2188: rollback oid 333 current snap is 228 2014-07-01 17:04:10,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 333 to 218 2014-07-01 17:04:10,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2189: snap_create 2014-07-01 17:04:10,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 751 (ObjNum 332 snap 0 seq_num 332) dirty exists 2014-07-01 17:04:11,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2188: done (1 left) 2014-07-01 17:04:11,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2189: done (0 left) 2014-07-01 17:04:11,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: write oid 61 current snap is 229 2014-07-01 17:04:11,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: seq_num 1126 ranges {577680=536394,1528466=623058,2875215=502465} 2014-07-01 17:04:11,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: writing mira0437250-61 from 577680 to 1114074 tid 1 2014-07-01 17:04:11,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: writing mira0437250-61 from 1528466 to 2151524 tid 2 2014-07-01 17:04:11,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: writing mira0437250-61 from 2875215 to 3377680 tid 3 2014-07-01 17:04:11,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: write oid 77 current snap is 229 2014-07-01 17:04:11,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: seq_num 1127 ranges {789261=659622,1909349=444677,2945239=425203,3589260=1} 2014-07-01 17:04:11,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: writing mira0437250-77 from 789261 to 1448883 tid 1 2014-07-01 17:04:11,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: writing mira0437250-77 from 1909349 to 2354026 tid 2 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: writing mira0437250-77 from 2945239 to 3370442 tid 3 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: writing mira0437250-77 from 3589260 to 3589261 tid 4 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2192: read oid 228 snap -1 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 1 to mira0437250-61 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 2 to mira0437250-61 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 3 to mira0437250-61 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 5 to mira0437250-61 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: finishing write tid 6 to mira0437250-61 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 870 (ObjNum 1126 snap 229 seq_num 1126) dirty exists 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 1 to mira0437250-77 2014-07-01 17:04:11,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 2 to mira0437250-77 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2190: done (2 left) 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2193: rollback oid 421 current snap is 229 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 421 to 211 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2194: rollback oid 121 current snap is 229 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 121 to 185 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2195: rollback oid 88 current snap is 229 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 88 to 224 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: read oid 180 snap 221 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: write oid 177 current snap is 229 2014-07-01 17:04:11,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: seq_num 1128 ranges {485515=782850,1738809=763449,2994017=291498} 2014-07-01 17:04:11,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: writing mira0437250-177 from 485515 to 1268365 tid 1 2014-07-01 17:04:11,768.768 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:04:11,768.768 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pg_num 56' 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: writing mira0437250-177 from 1738809 to 2502258 tid 2 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: writing mira0437250-177 from 2994017 to 3285515 tid 3 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: read oid 57 snap -1 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 988 (ObjNum 120 snap 0 seq_num 120) dirty exists 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 3 to mira0437250-77 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 4 to mira0437250-77 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 6 to mira0437250-77 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 1436 (ObjNum 983 snap 185 seq_num 983) dirty exists 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: finishing write tid 7 to mira0437250-77 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 757 (ObjNum 1127 snap 229 seq_num 1127) dirty exists 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 990 (ObjNum 1089 snap 218 seq_num 1089) dirty exists 2014-07-01 17:04:11,789.789 INFO:teuthology.task.rados.rados.0.mira043.stdout:2192: expect (ObjNum 227 snap 0 seq_num 227) 2014-07-01 17:04:12,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: finishing write tid 2 to mira0435601-4 2014-07-01 17:04:12,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: finishing write tid 4 to mira0435601-4 2014-07-01 17:04:12,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: finishing write tid 5 to mira0435601-4 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 239 (ObjNum 2036 snap 0 seq_num 2036) dirty exists 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 1 to mira0435601-165 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 2 to mira0435601-165 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 3 to mira0435601-165 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3271: done (11 left) 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: done (10 left) 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: done (9 left) 2014-07-01 17:04:12,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3275: done (8 left) 2014-07-01 17:04:12,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: done (7 left) 2014-07-01 17:04:12,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 21 current snap is 0 2014-07-01 17:04:12,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: seq_num 2042 ranges {7815168=491520,8306688=753664,9060352=49152} 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: writing mira0435601-21 from 7815168 to 8306688 tid 1 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: writing mira0435601-21 from 8306688 to 9060352 tid 2 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: writing mira0435601-21 from 9060352 to 9109504 tid 3 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: read oid 101 snap -1 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 4 to mira0435601-165 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 6 to mira0435601-165 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: finishing write tid 7 to mira0435601-165 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 165 v 285 (ObjNum 2037 snap 0 seq_num 2037) dirty exists 2014-07-01 17:04:12,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3278: expect (ObjNum 1489 snap 0 seq_num 1489) 2014-07-01 17:04:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: finishing write tid 1 to mira0437250-177 2014-07-01 17:04:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: finishing write tid 2 to mira0437250-177 2014-07-01 17:04:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: finishing write tid 3 to mira0437250-177 2014-07-01 17:04:12,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: finishing write tid 5 to mira0437250-177 2014-07-01 17:04:12,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: finishing write tid 6 to mira0437250-177 2014-07-01 17:04:12,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 409 (ObjNum 1128 snap 229 seq_num 1128) dirty exists 2014-07-01 17:04:12,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: expect (ObjNum 955 snap 175 seq_num 955) 2014-07-01 17:04:13,498.498 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pg_num to 56 2014-07-01 17:04:13,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:2191: done (7 left) 2014-07-01 17:04:13,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:2192: done (6 left) 2014-07-01 17:04:13,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2193: done (5 left) 2014-07-01 17:04:13,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2194: done (4 left) 2014-07-01 17:04:13,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2195: done (3 left) 2014-07-01 17:04:13,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2196: done (2 left) 2014-07-01 17:04:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2197: done (1 left) 2014-07-01 17:04:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2198: done (0 left) 2014-07-01 17:04:13,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2199: snap_remove snap 206 2014-07-01 17:04:14,515.515 INFO:teuthology.task.rados.rados.0.mira043.stdout:2199: done (0 left) 2014-07-01 17:04:14,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: read oid 171 snap -1 2014-07-01 17:04:14,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: snap_remove snap 215 2014-07-01 17:04:14,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: expect (ObjNum 170 snap 0 seq_num 170) 2014-07-01 17:04:14,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 1 to mira0435601-409 2014-07-01 17:04:14,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 2 to mira0435601-409 2014-07-01 17:04:14,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 3 to mira0435601-409 2014-07-01 17:04:14,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 5 to mira0435601-409 2014-07-01 17:04:14,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 6 to mira0435601-409 2014-07-01 17:04:14,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 367 (ObjNum 2038 snap 0 seq_num 2038) dirty exists 2014-07-01 17:04:14,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: finishing write tid 1 to mira0435601-94 2014-07-01 17:04:14,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: finishing write tid 3 to mira0435601-94 2014-07-01 17:04:14,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: finishing write tid 4 to mira0435601-94 2014-07-01 17:04:14,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 257 (ObjNum 2039 snap 0 seq_num 2039) dirty exists 2014-07-01 17:04:14,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 1 to mira0435601-257 2014-07-01 17:04:14,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 2 to mira0435601-257 2014-07-01 17:04:14,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 3 to mira0435601-257 2014-07-01 17:04:14,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 5 to mira0435601-257 2014-07-01 17:04:14,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 6 to mira0435601-257 2014-07-01 17:04:14,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 257 v 198 (ObjNum 2040 snap 0 seq_num 2040) dirty exists 2014-07-01 17:04:14,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: finishing write tid 1 to mira0435601-483 2014-07-01 17:04:14,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: finishing write tid 3 to mira0435601-483 2014-07-01 17:04:14,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: expect (ObjNum 2016 snap 0 seq_num 2016) 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: finishing write tid 4 to mira0435601-483 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 483 v 187 (ObjNum 2041 snap 0 seq_num 2041) dirty exists 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: finishing write tid 1 to mira0435601-21 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: finishing write tid 2 to mira0435601-21 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: finishing write tid 3 to mira0435601-21 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: finishing write tid 5 to mira0435601-21 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: finishing write tid 6 to mira0435601-21 2014-07-01 17:04:14,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 272 (ObjNum 2042 snap 0 seq_num 2042) dirty exists 2014-07-01 17:04:15,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: expect (ObjNum 1888 snap 0 seq_num 1888) 2014-07-01 17:04:16,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:2200: done (1 left) 2014-07-01 17:04:16,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2201: done (0 left) 2014-07-01 17:04:16,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2202: snap_remove snap 217 2014-07-01 17:04:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: done (8 left) 2014-07-01 17:04:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:3278: done (7 left) 2014-07-01 17:04:17,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: done (6 left) 2014-07-01 17:04:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: done (5 left) 2014-07-01 17:04:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: done (4 left) 2014-07-01 17:04:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: done (3 left) 2014-07-01 17:04:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: done (2 left) 2014-07-01 17:04:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: done (1 left) 2014-07-01 17:04:17,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: done (0 left) 2014-07-01 17:04:17,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 19 current snap is 0 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: seq_num 2043 ranges {2711552=659456} 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: writing mira0435601-19 from 2711552 to 3371008 tid 1 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: seq_num 2044 ranges {9908224=688128,10596352=565248,11161600=638976} 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: writing mira0435601-27 from 9908224 to 10596352 tid 1 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: writing mira0435601-27 from 10596352 to 11161600 tid 2 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: writing mira0435601-27 from 11161600 to 11800576 tid 3 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 399 current snap is 0 2014-07-01 17:04:17,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: seq_num 2045 ranges {360448=712704} 2014-07-01 17:04:17,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: writing mira0435601-399 from 360448 to 1073152 tid 1 2014-07-01 17:04:17,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: read oid 45 snap -1 2014-07-01 17:04:17,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 1 to mira0435601-19 2014-07-01 17:04:17,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 3 to mira0435601-19 2014-07-01 17:04:17,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 4 to mira0435601-19 2014-07-01 17:04:17,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 202 (ObjNum 2043 snap 0 seq_num 2043) dirty exists 2014-07-01 17:04:17,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 1 to mira0435601-27 2014-07-01 17:04:17,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 2 to mira0435601-27 2014-07-01 17:04:17,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 3 to mira0435601-27 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 5 to mira0435601-27 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: done (3 left) 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 87 current snap is 0 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: seq_num 2046 ranges {675840=585728} 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: writing mira0435601-87 from 675840 to 1261568 tid 1 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: read oid 431 snap -1 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3292: read oid 335 snap -1 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 6 to mira0435601-27 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 243 (ObjNum 2044 snap 0 seq_num 2044) dirty exists 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: done (5 left) 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 219 current snap is 0 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: seq_num 2047 ranges {0=507904,507904=561152,1069056=241664} 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: writing mira0435601-219 from 0 to 507904 tid 1 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: writing mira0435601-219 from 507904 to 1069056 tid 2 2014-07-01 17:04:17,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: writing mira0435601-219 from 1069056 to 1310720 tid 3 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 309 current snap is 0 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: seq_num 2048 ranges {2596864=131072} 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: writing mira0435601-309 from 2596864 to 2727936 tid 1 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3295: read oid 458 snap -1 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: finishing write tid 1 to mira0435601-399 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: finishing write tid 3 to mira0435601-399 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 474 current snap is 0 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: seq_num 2049 ranges {3596288=4096} 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: writing mira0435601-474 from 3596288 to 3600384 tid 1 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3297: read oid 196 snap -1 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3298: delete oid 247 current snap is 0 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: finishing write tid 4 to mira0435601-399 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 399 v 204 (ObjNum 2045 snap 0 seq_num 2045) dirty exists 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 1 to mira0435601-219 2014-07-01 17:04:17,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 2 to mira0435601-219 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 3 to mira0435601-219 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 5 to mira0435601-219 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 6 to mira0435601-219 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 371 (ObjNum 2047 snap 0 seq_num 2047) dirty exists 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 1 to mira0435601-309 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 3 to mira0435601-309 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: done (10 left) 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3292: done (9 left) 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: done (8 left) 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3298: done (7 left) 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 17:04:17,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: seq_num 2051 ranges {7614464=704512,8318976=401408} 2014-07-01 17:04:17,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: writing mira0435601-252 from 7614464 to 8318976 tid 1 2014-07-01 17:04:17,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: writing mira0435601-252 from 8318976 to 8720384 tid 2 2014-07-01 17:04:17,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3300: read oid 283 snap -1 2014-07-01 17:04:17,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: expect (ObjNum 1674 snap 0 seq_num 1674) 2014-07-01 17:04:17,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2202: done (0 left) 2014-07-01 17:04:17,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2203: rollback oid 224 current snap is 229 2014-07-01 17:04:17,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 224 to 228 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: read oid 116 snap 225 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: rollback oid 17 current snap is 229 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 17 to 197 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: snap_remove snap 199 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: expect (ObjNum 115 snap 0 seq_num 115) 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 17 v 421 (ObjNum 535 snap 14 seq_num 535) dirty exists 2014-07-01 17:04:17,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 1278 (ObjNum 655 snap 73 seq_num 655) dirty exists 2014-07-01 17:04:18,516.516 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:18,516.516 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:18,517.517 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:04:18,517.517 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:04:18,517.517 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:04:18,517.517 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:04:18,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2203: done (3 left) 2014-07-01 17:04:18,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2204: done (2 left) 2014-07-01 17:04:18,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2205: done (1 left) 2014-07-01 17:04:18,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2206: done (0 left) 2014-07-01 17:04:18,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: write oid 299 current snap is 229 2014-07-01 17:04:18,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: seq_num 1129 ranges {678921=400000} 2014-07-01 17:04:18,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: writing mira0437250-299 from 678921 to 1078921 tid 1 2014-07-01 17:04:18,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: snap_create 2014-07-01 17:04:18,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: finishing write tid 1 to mira0437250-299 2014-07-01 17:04:18,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: finishing write tid 3 to mira0437250-299 2014-07-01 17:04:18,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: finishing write tid 4 to mira0437250-299 2014-07-01 17:04:18,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 770 (ObjNum 1129 snap 229 seq_num 1129) dirty exists 2014-07-01 17:04:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:2207: done (1 left) 2014-07-01 17:04:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:2208: done (0 left) 2014-07-01 17:04:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: write oid 16 current snap is 230 2014-07-01 17:04:19,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: seq_num 1130 ranges {774150=1} 2014-07-01 17:04:19,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: writing mira0437250-16 from 774150 to 774151 tid 1 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: read oid 419 snap -1 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: read oid 299 snap -1 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2212: rollback oid 34 current snap is 230 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 34 to 219 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: write oid 114 current snap is 230 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: seq_num 1131 ranges {470403=1} 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: writing mira0437250-114 from 470403 to 470404 tid 1 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: write oid 95 current snap is 230 2014-07-01 17:04:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: seq_num 1132 ranges {530562=707188,1730561=1} 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: writing mira0437250-95 from 530562 to 1237750 tid 1 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: writing mira0437250-95 from 1730561 to 1730562 tid 2 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2215: snap_create 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: expect (ObjNum 418 snap 0 seq_num 418) 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: finishing write tid 1 to mira0437250-16 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: finishing write tid 3 to mira0437250-16 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: finishing write tid 4 to mira0437250-16 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 749 (ObjNum 1130 snap 230 seq_num 1130) dirty exists 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: expect (ObjNum 1129 snap 229 seq_num 1129) 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 4 to mira0435601-309 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 289 (ObjNum 2048 snap 0 seq_num 2048) dirty exists 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: finishing write tid 1 to mira0435601-474 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: finishing write tid 3 to mira0435601-474 2014-07-01 17:04:19,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:3295: expect (ObjNum 1232 snap 0 seq_num 1232) 2014-07-01 17:04:19,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 822 (ObjNum 825 snap 132 seq_num 825) dirty exists 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: finishing write tid 1 to mira0437250-114 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: finishing write tid 3 to mira0437250-114 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: finishing write tid 4 to mira0437250-114 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 1198 (ObjNum 1131 snap 230 seq_num 1131) dirty exists 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: finishing write tid 1 to mira0437250-95 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: finishing write tid 2 to mira0437250-95 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: finishing write tid 4 to mira0437250-95 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: finishing write tid 5 to mira0437250-95 2014-07-01 17:04:19,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 721 (ObjNum 1132 snap 230 seq_num 1132) dirty exists 2014-07-01 17:04:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2209: done (6 left) 2014-07-01 17:04:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2210: done (5 left) 2014-07-01 17:04:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2211: done (4 left) 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2212: done (3 left) 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2213: done (2 left) 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2214: done (1 left) 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2215: done (0 left) 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: read oid 249 snap -1 2014-07-01 17:04:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2217: read oid 76 snap -1 2014-07-01 17:04:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2218: delete oid 356 current snap is 231 2014-07-01 17:04:20,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2218: done (2 left) 2014-07-01 17:04:20,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2219: snap_remove snap 188 2014-07-01 17:04:20,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2217: expect (ObjNum 75 snap 0 seq_num 75) 2014-07-01 17:04:21,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:2216: done (2 left) 2014-07-01 17:04:21,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:2217: done (1 left) 2014-07-01 17:04:21,583.583 INFO:teuthology.task.rados.rados.0.mira043.stdout:2219: done (0 left) 2014-07-01 17:04:21,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: snap_remove snap 225 2014-07-01 17:04:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: finishing write tid 1 to mira0435601-87 2014-07-01 17:04:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: finishing write tid 3 to mira0435601-87 2014-07-01 17:04:22,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: finishing write tid 4 to mira0435601-474 2014-07-01 17:04:22,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 210 (ObjNum 2049 snap 0 seq_num 2049) dirty exists 2014-07-01 17:04:22,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: expect (ObjNum 1108 snap 0 seq_num 1108) 2014-07-01 17:04:22,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2220: done (0 left) 2014-07-01 17:04:22,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: snap_remove snap 219 2014-07-01 17:04:23,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2221: done (0 left) 2014-07-01 17:04:23,607.607 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: snap_create 2014-07-01 17:04:23,947.947 INFO:teuthology.task.thrashosds.thrasher:in_osds: [3, 5, 4, 2, 0, 1] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:23,947.947 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:23,947.947 INFO:teuthology.task.thrashosds.thrasher:Removing osd 3, in_osds are: [3, 5, 4, 2, 0, 1] 2014-07-01 17:04:23,948.948 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 3' 2014-07-01 17:04:24,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:2222: done (0 left) 2014-07-01 17:04:24,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: read oid 77 snap -1 2014-07-01 17:04:24,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: write oid 312 current snap is 232 2014-07-01 17:04:24,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: seq_num 1134 ranges {571646=685868,1934259=237387} 2014-07-01 17:04:24,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: writing mira0437250-312 from 571646 to 1257514 tid 1 2014-07-01 17:04:24,670.670 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.3. 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: writing mira0437250-312 from 1934259 to 2171646 tid 2 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2225: rollback oid 468 current snap is 232 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 468 to 205 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: rollback oid 293 current snap is 232 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 293 to 218 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: read oid 448 snap -1 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: write oid 29 current snap is 232 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: seq_num 1135 ranges {514427=657673,1611271=791711,2914426=1} 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0437250-29 from 514427 to 1172100 tid 1 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0437250-29 from 1611271 to 2402982 tid 2 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: writing mira0437250-29 from 2914426 to 2914427 tid 3 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2229: delete oid 205 current snap is 232 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: expect (ObjNum 704 snap 93 seq_num 704) 2014-07-01 17:04:24,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 1077 (ObjNum 1067 snap 213 seq_num 1067) dirty exists 2014-07-01 17:04:24,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: expect (ObjNum 1127 snap 229 seq_num 1127) 2014-07-01 17:04:26,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: finishing write tid 1 to mira0437250-312 2014-07-01 17:04:26,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: finishing write tid 2 to mira0437250-312 2014-07-01 17:04:26,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: finishing write tid 4 to mira0437250-312 2014-07-01 17:04:26,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 1 to mira0437250-29 2014-07-01 17:04:26,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: finishing write tid 5 to mira0437250-312 2014-07-01 17:04:26,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 1253 (ObjNum 1134 snap 232 seq_num 1134) dirty exists 2014-07-01 17:04:26,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 2 to mira0437250-29 2014-07-01 17:04:26,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 3 to mira0437250-29 2014-07-01 17:04:26,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 5 to mira0437250-29 2014-07-01 17:04:26,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: finishing write tid 6 to mira0437250-29 2014-07-01 17:04:26,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 787 (ObjNum 1135 snap 232 seq_num 1135) dirty exists 2014-07-01 17:04:26,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2223: done (6 left) 2014-07-01 17:04:26,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2224: done (5 left) 2014-07-01 17:04:26,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2226: done (4 left) 2014-07-01 17:04:26,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2227: done (3 left) 2014-07-01 17:04:26,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2228: done (2 left) 2014-07-01 17:04:26,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2229: done (1 left) 2014-07-01 17:04:26,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2230: read oid 115 snap 223 2014-07-01 17:04:26,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: read oid 431 snap 231 2014-07-01 17:04:26,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: write oid 136 current snap is 232 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: seq_num 1137 ranges {670839=754753,2204463=683525,3682998=187841} 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0437250-136 from 670839 to 1425592 tid 1 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0437250-136 from 2204463 to 2887988 tid 2 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: writing mira0437250-136 from 3682998 to 3870839 tid 3 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: write oid 186 current snap is 232 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: seq_num 1138 ranges {615668=1} 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: writing mira0437250-186 from 615668 to 615669 tid 1 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2234: rollback oid 79 current snap is 232 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 79 to 218 2014-07-01 17:04:26,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:2230: expect (ObjNum 1098 snap 220 seq_num 1098) 2014-07-01 17:04:26,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2230: done (5 left) 2014-07-01 17:04:26,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2231: done (4 left) 2014-07-01 17:04:26,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2235: snap_create 2014-07-01 17:04:26,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2235: done (4 left) 2014-07-01 17:04:26,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: snap_create 2014-07-01 17:04:27,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: finishing write tid 4 to mira0435601-87 2014-07-01 17:04:27,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 87 v 224 (ObjNum 2046 snap 0 seq_num 2046) dirty exists 2014-07-01 17:04:27,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:3297: expect (ObjNum 1972 snap 0 seq_num 1972) 2014-07-01 17:04:27,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 468 v 698 (ObjNum 467 snap 0 seq_num 467) dirty exists 2014-07-01 17:04:27,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 1 to mira0437250-136 2014-07-01 17:04:27,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 2 to mira0437250-136 2014-07-01 17:04:27,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 3 to mira0437250-136 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 5 to mira0437250-136 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 1304 (ObjNum 715 snap 96 seq_num 715) dirty exists 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: finishing write tid 6 to mira0437250-136 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 1049 (ObjNum 1137 snap 232 seq_num 1137) dirty exists 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2225: done (4 left) 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2232: done (3 left) 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2234: done (2 left) 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2236: done (1 left) 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2237: delete oid 465 current snap is 234 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2237: done (1 left) 2014-07-01 17:04:27,938.938 INFO:teuthology.task.rados.rados.0.mira043.stdout:2238: delete oid 193 current snap is 234 2014-07-01 17:04:27,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2238: done (1 left) 2014-07-01 17:04:27,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: write oid 300 current snap is 234 2014-07-01 17:04:27,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: seq_num 1141 ranges {716910=678067,2039424=277486} 2014-07-01 17:04:27,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: writing mira0437250-300 from 716910 to 1394977 tid 1 2014-07-01 17:04:27,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: writing mira0437250-300 from 2039424 to 2316910 tid 2 2014-07-01 17:04:28,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2240: snap_create 2014-07-01 17:04:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: finishing write tid 1 to mira0437250-186 2014-07-01 17:04:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: finishing write tid 3 to mira0437250-186 2014-07-01 17:04:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: finishing write tid 4 to mira0437250-186 2014-07-01 17:04:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 903 (ObjNum 1138 snap 232 seq_num 1138) dirty exists 2014-07-01 17:04:28,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: finishing write tid 1 to mira0437250-300 2014-07-01 17:04:28,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: finishing write tid 2 to mira0437250-300 2014-07-01 17:04:28,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: finishing write tid 4 to mira0437250-300 2014-07-01 17:04:28,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: finishing write tid 5 to mira0437250-300 2014-07-01 17:04:28,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 1472 (ObjNum 1141 snap 234 seq_num 1141) dirty exists 2014-07-01 17:04:28,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2233: done (2 left) 2014-07-01 17:04:28,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2239: done (1 left) 2014-07-01 17:04:28,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2240: done (0 left) 2014-07-01 17:04:28,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: write oid 259 current snap is 235 2014-07-01 17:04:28,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: seq_num 1142 ranges {660715=684943,2036179=785925,3060714=1} 2014-07-01 17:04:28,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: writing mira0437250-259 from 660715 to 1345658 tid 1 2014-07-01 17:04:28,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: writing mira0437250-259 from 2036179 to 2822104 tid 2 2014-07-01 17:04:29,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: writing mira0437250-259 from 3060714 to 3060715 tid 3 2014-07-01 17:04:29,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: read oid 217 snap -1 2014-07-01 17:04:29,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2243: delete oid 272 current snap is 235 2014-07-01 17:04:29,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: expect (ObjNum 826 snap 132 seq_num 826) 2014-07-01 17:04:29,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: finishing write tid 1 to mira0437250-259 2014-07-01 17:04:29,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: finishing write tid 2 to mira0437250-259 2014-07-01 17:04:29,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: finishing write tid 3 to mira0437250-259 2014-07-01 17:04:29,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: finishing write tid 5 to mira0437250-259 2014-07-01 17:04:29,614.614 INFO:teuthology.task.rados.rados.0.mira043.stdout:2242: done (2 left) 2014-07-01 17:04:29,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2243: done (1 left) 2014-07-01 17:04:29,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: snap_create 2014-07-01 17:04:29,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: finishing write tid 6 to mira0437250-259 2014-07-01 17:04:29,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 763 (ObjNum 1142 snap 235 seq_num 1142) dirty exists 2014-07-01 17:04:29,686.686 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 2, 0, 1] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:29,686.686 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:29,686.686 INFO:teuthology.task.thrashosds.thrasher:Adding osd 3 2014-07-01 17:04:29,686.686 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 3' 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 1 to mira0435601-252 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 2 to mira0435601-252 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 4 to mira0435601-252 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 5 to mira0435601-252 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 364 (ObjNum 2051 snap 0 seq_num 2051) dirty exists 2014-07-01 17:04:29,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: done (8 left) 2014-07-01 17:04:29,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: done (7 left) 2014-07-01 17:04:29,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: done (6 left) 2014-07-01 17:04:29,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: done (5 left) 2014-07-01 17:04:29,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3295: done (4 left) 2014-07-01 17:04:29,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: done (3 left) 2014-07-01 17:04:29,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3297: done (2 left) 2014-07-01 17:04:29,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: done (1 left) 2014-07-01 17:04:29,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3300: done (0 left) 2014-07-01 17:04:29,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 301 current snap is 0 2014-07-01 17:04:29,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: seq_num 2052 ranges {1564672=626688,2191360=774144,2965504=430080} 2014-07-01 17:04:29,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: writing mira0435601-301 from 1564672 to 2191360 tid 1 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: writing mira0435601-301 from 2191360 to 2965504 tid 2 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: writing mira0435601-301 from 2965504 to 3395584 tid 3 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: read oid 102 snap -1 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 34 current snap is 0 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: seq_num 2053 ranges {3772416=499712,4272128=602112,4874240=602112} 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0435601-34 from 3772416 to 4272128 tid 1 2014-07-01 17:04:29,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0435601-34 from 4272128 to 4874240 tid 2 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0435601-34 from 4874240 to 5476352 tid 3 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 148 current snap is 0 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: seq_num 2054 ranges {2260992=524288} 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2241: done (1 left) 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2244: done (0 left) 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: read oid 177 snap -1 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: write oid 59 current snap is 236 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: seq_num 1144 ranges {700775=781000,2228348=712812,3473400=27375} 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: writing mira0437250-59 from 700775 to 1481775 tid 1 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: writing mira0437250-59 from 2228348 to 2941160 tid 2 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: writing mira0437250-59 from 3473400 to 3500775 tid 3 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: write oid 431 current snap is 236 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: seq_num 1145 ranges {796005=752804,2057808=631583,3196004=1} 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: writing mira0437250-431 from 796005 to 1548809 tid 1 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: writing mira0437250-431 from 2057808 to 2689391 tid 2 2014-07-01 17:04:29,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: writing mira0437250-431 from 3196004 to 3196005 tid 3 2014-07-01 17:04:30,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: write oid 128 current snap is 236 2014-07-01 17:04:30,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: seq_num 1146 ranges {705762=759680,1505761=1} 2014-07-01 17:04:30,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: writing mira0437250-128 from 705762 to 1465442 tid 1 2014-07-01 17:04:30,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: writing mira0437250-128 from 1505761 to 1505762 tid 2 2014-07-01 17:04:30,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: read oid 6 snap 229 2014-07-01 17:04:30,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: expect (ObjNum 1128 snap 229 seq_num 1128) 2014-07-01 17:04:30,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: writing mira0435601-148 from 2260992 to 2785280 tid 1 2014-07-01 17:04:30,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3305: read oid 438 snap -1 2014-07-01 17:04:30,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: expect (ObjNum 1253 snap 0 seq_num 1253) 2014-07-01 17:04:30,888.888 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.3. 2014-07-01 17:04:30,898.898 INFO:teuthology.task.thrashosds.thrasher:Added osd 3 2014-07-01 17:04:31,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: finishing write tid 1 to mira0437250-431 2014-07-01 17:04:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 1 to mira0437250-128 2014-07-01 17:04:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: finishing write tid 2 to mira0437250-431 2014-07-01 17:04:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 2 to mira0437250-128 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 4 to mira0437250-128 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: finishing write tid 3 to mira0437250-431 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: finishing write tid 5 to mira0437250-431 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: finishing write tid 6 to mira0437250-431 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 1081 (ObjNum 1145 snap 236 seq_num 1145) dirty exists 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: finishing write tid 5 to mira0437250-128 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 128 v 760 (ObjNum 1146 snap 236 seq_num 1146) dirty exists 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2245: done (4 left) 2014-07-01 17:04:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:2247: done (3 left) 2014-07-01 17:04:31,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2248: done (2 left) 2014-07-01 17:04:31,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: write oid 402 current snap is 236 2014-07-01 17:04:31,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: seq_num 1147 ranges {400992=732509,1200991=1} 2014-07-01 17:04:31,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: writing mira0437250-402 from 400992 to 1133501 tid 1 2014-07-01 17:04:31,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: writing mira0437250-402 from 1200991 to 1200992 tid 2 2014-07-01 17:04:31,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: read oid 221 snap 235 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: write oid 127 current snap is 236 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: seq_num 1148 ranges {497246=400000} 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: writing mira0437250-127 from 497246 to 897246 tid 1 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2253: delete oid 322 current snap is 236 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: finishing write tid 1 to mira0435601-301 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: finishing write tid 2 to mira0435601-301 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: done (4 left) 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 486 current snap is 0 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: seq_num 2055 ranges {4063232=397312} 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: writing mira0435601-486 from 4063232 to 4460544 tid 1 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 81 current snap is 0 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: seq_num 2056 ranges {1310720=720896,2031616=401408} 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: writing mira0435601-81 from 1310720 to 2031616 tid 1 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: writing mira0435601-81 from 2031616 to 2433024 tid 2 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3308: read oid 42 snap -1 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: read oid 126 snap -1 2014-07-01 17:04:31,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3310: read oid 18 snap -1 2014-07-01 17:04:31,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3311: read oid 445 snap -1 2014-07-01 17:04:31,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 17:04:31,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: seq_num 2057 ranges {4165632=745472,4911104=585728,5496832=782336,6279168=237568} 2014-07-01 17:04:31,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: writing mira0435601-327 from 4165632 to 4911104 tid 1 2014-07-01 17:04:31,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: writing mira0435601-327 from 4911104 to 5496832 tid 2 2014-07-01 17:04:31,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: expect (ObjNum 729 snap 98 seq_num 729) 2014-07-01 17:04:31,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: writing mira0435601-327 from 5496832 to 6279168 tid 3 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: writing mira0435601-327 from 6279168 to 6516736 tid 4 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3313: read oid 306 snap -1 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 118 current snap is 0 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: seq_num 2058 ranges {1785856=618496,2404352=520192,2924544=659456,3584000=425984} 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: writing mira0435601-118 from 1785856 to 2404352 tid 1 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: writing mira0435601-118 from 2404352 to 2924544 tid 2 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: writing mira0435601-118 from 2924544 to 3584000 tid 3 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: writing mira0435601-118 from 3584000 to 4009984 tid 4 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3315: read oid 358 snap -1 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: read oid 170 snap -1 2014-07-01 17:04:31,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3317: delete oid 293 current snap is 0 2014-07-01 17:04:31,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 1 to mira0437250-402 2014-07-01 17:04:31,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 2 to mira0437250-402 2014-07-01 17:04:31,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 1 to mira0437250-127 2014-07-01 17:04:31,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 4 to mira0437250-402 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 3 to mira0437250-127 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: finishing write tid 5 to mira0437250-402 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 854 (ObjNum 1147 snap 236 seq_num 1147) dirty exists 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: finishing write tid 4 to mira0437250-127 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 795 (ObjNum 1148 snap 236 seq_num 1148) dirty exists 2014-07-01 17:04:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: finishing write tid 1 to mira0437250-59 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: finishing write tid 2 to mira0437250-59 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: finishing write tid 3 to mira0437250-59 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: finishing write tid 5 to mira0437250-59 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: finishing write tid 6 to mira0437250-59 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 836 (ObjNum 1144 snap 236 seq_num 1144) dirty exists 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3313: done (15 left) 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3317: done (14 left) 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: read oid 452 snap -1 2014-07-01 17:04:32,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: expect (ObjNum 2002 snap 0 seq_num 2002) 2014-07-01 17:04:33,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 1 to mira0435601-148 2014-07-01 17:04:33,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 3 to mira0435601-148 2014-07-01 17:04:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: finishing write tid 3 to mira0435601-301 2014-07-01 17:04:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 4 to mira0435601-148 2014-07-01 17:04:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 366 (ObjNum 2054 snap 0 seq_num 2054) dirty exists 2014-07-01 17:04:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: finishing write tid 5 to mira0435601-301 2014-07-01 17:04:33,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: finishing write tid 6 to mira0435601-301 2014-07-01 17:04:33,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 308 (ObjNum 2052 snap 0 seq_num 2052) dirty exists 2014-07-01 17:04:33,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: expect (ObjNum 1404 snap 0 seq_num 1404) 2014-07-01 17:04:35,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:2246: done (5 left) 2014-07-01 17:04:35,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2249: done (4 left) 2014-07-01 17:04:35,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2250: done (3 left) 2014-07-01 17:04:35,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2251: done (2 left) 2014-07-01 17:04:35,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2252: done (1 left) 2014-07-01 17:04:35,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2253: done (0 left) 2014-07-01 17:04:35,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: rollback oid 176 current snap is 236 2014-07-01 17:04:35,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 176 to 211 2014-07-01 17:04:35,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2255: delete oid 355 current snap is 236 2014-07-01 17:04:35,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2255: done (1 left) 2014-07-01 17:04:35,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2256: snap_create 2014-07-01 17:04:35,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2256: done (1 left) 2014-07-01 17:04:35,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: write oid 444 current snap is 237 2014-07-01 17:04:35,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: seq_num 1151 ranges {597463=655822,1817161=707026,2597462=1} 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0437250-444 from 597463 to 1253285 tid 1 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0437250-444 from 1817161 to 2524187 tid 2 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: writing mira0437250-444 from 2597462 to 2597463 tid 3 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2258: rollback oid 316 current snap is 237 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 316 to 229 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2259: read oid 445 snap -1 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: rollback oid 32 current snap is 237 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 32 to 224 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: read oid 246 snap -1 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: write oid 17 current snap is 237 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: seq_num 1152 ranges {389844=1} 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: writing mira0437250-17 from 389844 to 389845 tid 1 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: read oid 421 snap -1 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: write oid 105 current snap is 237 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: seq_num 1153 ranges {86098=1} 2014-07-01 17:04:35,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: writing mira0437250-105 from 86098 to 86099 tid 1 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: read oid 229 snap -1 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2266: read oid 38 snap 236 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: write oid 331 current snap is 237 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: seq_num 1154 ranges {582352=758755,2007109=551660,3011158=655460,3782351=1} 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0437250-331 from 582352 to 1341107 tid 1 2014-07-01 17:04:35,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0437250-331 from 2007109 to 2558769 tid 2 2014-07-01 17:04:35,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0437250-331 from 3011158 to 3666618 tid 3 2014-07-01 17:04:35,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: writing mira0437250-331 from 3782351 to 3782352 tid 4 2014-07-01 17:04:35,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2268: rollback oid 497 current snap is 237 2014-07-01 17:04:35,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 497 to 224 2014-07-01 17:04:35,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: finishing write tid 1 to mira0437250-105 2014-07-01 17:04:35,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: expect (ObjNum 1082 snap 217 seq_num 1082) 2014-07-01 17:04:35,899.899 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 2, 0, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:35,899.899 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:35,899.899 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 17:04:35,899.899 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:04:36,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 32 v 0 (ObjNum 1 snap 1 seq_num 31512032) dirty dne 2014-07-01 17:04:36,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: finishing write tid 3 to mira0437250-105 2014-07-01 17:04:36,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: finishing write tid 4 to mira0437250-105 2014-07-01 17:04:36,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 1084 (ObjNum 1153 snap 237 seq_num 1153) dirty exists 2014-07-01 17:04:36,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:2259: expect (ObjNum 444 snap 0 seq_num 444) 2014-07-01 17:04:36,663.663 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:04:36,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 1 to mira0437250-444 2014-07-01 17:04:36,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 0 (ObjNum 42891744 snap 0 seq_num 0) dirty dne 2014-07-01 17:04:36,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 2 to mira0437250-444 2014-07-01 17:04:36,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 3 to mira0437250-444 2014-07-01 17:04:36,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 5 to mira0437250-444 2014-07-01 17:04:36,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: expect (ObjNum 983 snap 185 seq_num 983) 2014-07-01 17:04:37,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: expect (ObjNum 824 snap 132 seq_num 824) 2014-07-01 17:04:37,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3315: expect (ObjNum 1610 snap 0 seq_num 1610) 2014-07-01 17:04:38,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: finishing write tid 6 to mira0437250-444 2014-07-01 17:04:38,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 686 (ObjNum 1151 snap 237 seq_num 1151) dirty exists 2014-07-01 17:04:38,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 0 (ObjNum 17366528 snap 0 seq_num 0) dirty dne 2014-07-01 17:04:38,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:2257: done (12 left) 2014-07-01 17:04:38,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2258: done (11 left) 2014-07-01 17:04:38,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2259: done (10 left) 2014-07-01 17:04:38,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2260: done (9 left) 2014-07-01 17:04:38,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2261: done (8 left) 2014-07-01 17:04:38,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:2263: done (7 left) 2014-07-01 17:04:38,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2264: done (6 left) 2014-07-01 17:04:38,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2265: done (5 left) 2014-07-01 17:04:38,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2268: done (4 left) 2014-07-01 17:04:38,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2269: rollback oid 173 current snap is 237 2014-07-01 17:04:38,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 173 to 185 2014-07-01 17:04:38,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: rollback oid 450 current snap is 237 2014-07-01 17:04:38,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 450 to 224 2014-07-01 17:04:38,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: snap_remove snap 228 2014-07-01 17:04:39,898.898 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 1 to mira0435601-34 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 1 to mira0435601-81 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 2 to mira0435601-81 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 4 to mira0435601-81 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 2 to mira0435601-34 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 3 to mira0435601-34 2014-07-01 17:04:39,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: done (14 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: done (13 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3305: done (12 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: done (11 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3310: done (10 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3315: done (9 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: done (8 left) 2014-07-01 17:04:39,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: done (7 left) 2014-07-01 17:04:39,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 218 current snap is 0 2014-07-01 17:04:39,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: seq_num 2060 ranges {1687552=516096,2203648=401408} 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: writing mira0435601-218 from 1687552 to 2203648 tid 1 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: writing mira0435601-218 from 2203648 to 2605056 tid 2 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3320: read oid 401 snap -1 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 5 to mira0435601-34 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 1 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 2 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 3 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 4 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 6 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 5 to mira0435601-81 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 81 v 331 (ObjNum 2056 snap 0 seq_num 2056) dirty exists 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: finishing write tid 7 to mira0435601-327 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 278 (ObjNum 2057 snap 0 seq_num 2057) dirty exists 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 6 to mira0435601-34 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 273 (ObjNum 2053 snap 0 seq_num 2053) dirty exists 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 1 to mira0435601-118 2014-07-01 17:04:40,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 2 to mira0435601-118 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 3 to mira0435601-118 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 4 to mira0435601-118 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: done (8 left) 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: done (7 left) 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: done (6 left) 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3321: read oid 296 snap -1 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 6 to mira0435601-118 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: finishing write tid 7 to mira0435601-118 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 295 (ObjNum 2058 snap 0 seq_num 2058) dirty exists 2014-07-01 17:04:40,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3308: expect (ObjNum 1969 snap 0 seq_num 1969) 2014-07-01 17:04:40,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 1474 (ObjNum 172 snap 0 seq_num 172) dirty exists 2014-07-01 17:04:40,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 450 v 1306 (ObjNum 449 snap 0 seq_num 449) dirty exists 2014-07-01 17:04:40,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: finishing write tid 1 to mira0437250-17 2014-07-01 17:04:40,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: finishing write tid 3 to mira0437250-17 2014-07-01 17:04:40,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: finishing write tid 4 to mira0437250-17 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 17 v 452 (ObjNum 1152 snap 237 seq_num 1152) dirty exists 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2262: done (6 left) 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2269: done (5 left) 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2270: done (4 left) 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2271: done (3 left) 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2272: delete oid 389 current snap is 237 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2272: done (3 left) 2014-07-01 17:04:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: delete oid 204 current snap is 237 2014-07-01 17:04:41,474.474 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 17:04:41,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2266: expect (ObjNum 37 snap 0 seq_num 37) 2014-07-01 17:04:42,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: finishing write tid 1 to mira0435601-486 2014-07-01 17:04:42,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: finishing write tid 3 to mira0435601-486 2014-07-01 17:04:42,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3320: expect (ObjNum 1346 snap 0 seq_num 1346) 2014-07-01 17:04:42,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 1 to mira0437250-331 2014-07-01 17:04:42,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 2 to mira0437250-331 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 3 to mira0437250-331 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 4 to mira0437250-331 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 6 to mira0437250-331 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: finishing write tid 7 to mira0437250-331 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 916 (ObjNum 1154 snap 237 seq_num 1154) dirty exists 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 1208 (ObjNum 830 snap 134 seq_num 830) dirty exists 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2254: done (3 left) 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2266: done (2 left) 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2267: done (1 left) 2014-07-01 17:04:42,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2273: done (0 left) 2014-07-01 17:04:42,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: read oid 101 snap -1 2014-07-01 17:04:42,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2275: snap_create 2014-07-01 17:04:42,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3311: expect (ObjNum 1937 snap 0 seq_num 1937) 2014-07-01 17:04:43,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2275: done (1 left) 2014-07-01 17:04:43,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2276: delete oid 173 current snap is 238 2014-07-01 17:04:43,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2276: done (1 left) 2014-07-01 17:04:43,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: snap_remove snap 205 2014-07-01 17:04:44,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2277: done (1 left) 2014-07-01 17:04:44,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: write oid 428 current snap is 238 2014-07-01 17:04:44,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: seq_num 1158 ranges {627399=433493,1592470=583864,2227398=1} 2014-07-01 17:04:44,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0437250-428 from 627399 to 1060892 tid 1 2014-07-01 17:04:44,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0437250-428 from 1592470 to 2176334 tid 2 2014-07-01 17:04:44,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: writing mira0437250-428 from 2227398 to 2227399 tid 3 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: write oid 224 current snap is 238 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: seq_num 1159 ranges {687557=405297,1518058=654059,2725892=714692,3487556=1} 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: writing mira0437250-224 from 687557 to 1092854 tid 1 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: writing mira0437250-224 from 1518058 to 2172117 tid 2 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: writing mira0437250-224 from 2725892 to 3440584 tid 3 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: writing mira0437250-224 from 3487556 to 3487557 tid 4 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2280: read oid 21 snap -1 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2281: rollback oid 363 current snap is 238 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 363 to 237 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: read oid 418 snap -1 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: rollback oid 490 current snap is 238 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 490 to 229 2014-07-01 17:04:44,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: write oid 177 current snap is 238 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: seq_num 1160 ranges {782787=528525,1898942=475679,2795918=386869} 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: writing mira0437250-177 from 782787 to 1311312 tid 1 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: writing mira0437250-177 from 1898942 to 2374621 tid 2 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: writing mira0437250-177 from 2795918 to 3182787 tid 3 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: snap_remove snap 221 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 1126 (ObjNum 362 snap 0 seq_num 362) dirty exists 2014-07-01 17:04:44,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: expect (ObjNum 417 snap 0 seq_num 417) 2014-07-01 17:04:45,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: finishing write tid 4 to mira0435601-486 2014-07-01 17:04:45,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 239 (ObjNum 2055 snap 0 seq_num 2055) dirty exists 2014-07-01 17:04:45,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3321: expect (ObjNum 1808 snap 0 seq_num 1808) 2014-07-01 17:04:45,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 1 to mira0435601-218 2014-07-01 17:04:45,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 2 to mira0435601-218 2014-07-01 17:04:45,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 4 to mira0435601-218 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 5 to mira0435601-218 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 218 v 269 (ObjNum 2060 snap 0 seq_num 2060) dirty exists 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: done (6 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3308: done (5 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3311: done (4 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: done (3 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: done (2 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3320: done (1 left) 2014-07-01 17:04:45,613.613 INFO:teuthology.task.rados.rados.0.mira043.stdout:3321: done (0 left) 2014-07-01 17:04:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3322: read oid 150 snap -1 2014-07-01 17:04:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 365 current snap is 0 2014-07-01 17:04:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: seq_num 2061 ranges {360448=438272,798720=409600,1208320=430080,1638400=364544} 2014-07-01 17:04:45,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0435601-365 from 360448 to 798720 tid 1 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0435601-365 from 798720 to 1208320 tid 2 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0435601-365 from 1208320 to 1638400 tid 3 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0435601-365 from 1638400 to 2002944 tid 4 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 407 current snap is 0 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: seq_num 2062 ranges {1474560=462848} 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: writing mira0435601-407 from 1474560 to 1937408 tid 1 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 261 current snap is 0 2014-07-01 17:04:45,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: seq_num 2063 ranges {6668288=335872} 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: writing mira0435601-261 from 6668288 to 7004160 tid 1 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 201 current snap is 0 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: seq_num 2064 ranges {0=208896} 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: writing mira0435601-201 from 0 to 208896 tid 1 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3327: read oid 317 snap -1 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 413 current snap is 0 2014-07-01 17:04:45,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: seq_num 2065 ranges {4149248=684032,4833280=790528,5623808=675840,6299648=139264} 2014-07-01 17:04:45,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: writing mira0435601-413 from 4149248 to 4833280 tid 1 2014-07-01 17:04:45,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: writing mira0435601-413 from 4833280 to 5623808 tid 2 2014-07-01 17:04:45,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: writing mira0435601-413 from 5623808 to 6299648 tid 3 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: writing mira0435601-413 from 6299648 to 6438912 tid 4 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: read oid 423 snap -1 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: read oid 227 snap -1 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: seq_num 2066 ranges {8486912=557056,9043968=724992,9768960=552960,10321920=327680} 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: writing mira0435601-391 from 8486912 to 9043968 tid 1 2014-07-01 17:04:45,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: writing mira0435601-391 from 9043968 to 9768960 tid 2 2014-07-01 17:04:45,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: writing mira0435601-391 from 9768960 to 10321920 tid 3 2014-07-01 17:04:45,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: writing mira0435601-391 from 10321920 to 10649600 tid 4 2014-07-01 17:04:45,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: read oid 278 snap -1 2014-07-01 17:04:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 225 current snap is 0 2014-07-01 17:04:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: seq_num 2067 ranges {0=430080,430080=557056,987136=778240,1765376=270336} 2014-07-01 17:04:45,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: writing mira0435601-225 from 0 to 430080 tid 1 2014-07-01 17:04:45,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: writing mira0435601-225 from 430080 to 987136 tid 2 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: writing mira0435601-225 from 987136 to 1765376 tid 3 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: writing mira0435601-225 from 1765376 to 2035712 tid 4 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3334: read oid 385 snap -1 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3335: read oid 272 snap -1 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3336: read oid 466 snap -1 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 342 current snap is 0 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: seq_num 2068 ranges {0=454656,454656=401408} 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: writing mira0435601-342 from 0 to 454656 tid 1 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: writing mira0435601-342 from 454656 to 856064 tid 2 2014-07-01 17:04:46,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 17:04:46,484.484 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 2, 0, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:46,484.484 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:46,484.484 INFO:teuthology.task.thrashosds.thrasher:Removing osd 2, in_osds are: [5, 4, 2, 0, 1, 3] 2014-07-01 17:04:46,484.484 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 2' 2014-07-01 17:04:46,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:2280: done (8 left) 2014-07-01 17:04:46,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2281: done (7 left) 2014-07-01 17:04:46,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2282: done (6 left) 2014-07-01 17:04:46,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2285: done (5 left) 2014-07-01 17:04:46,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: rollback oid 336 current snap is 238 2014-07-01 17:04:46,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 336 to 237 2014-07-01 17:04:46,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: read oid 169 snap -1 2014-07-01 17:04:46,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2288: snap_create 2014-07-01 17:04:47,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 0 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:04:47,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 1 to mira0437250-224 2014-07-01 17:04:47,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:3322: done (15 left) 2014-07-01 17:04:47,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:3338: read oid 381 snap -1 2014-07-01 17:04:47,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 2 to mira0437250-224 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 3 to mira0437250-224 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 4 to mira0437250-224 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 6 to mira0437250-224 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 17:04:47,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 1 to mira0435601-365 2014-07-01 17:04:47,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 2 to mira0435601-365 2014-07-01 17:04:47,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 3 to mira0435601-365 2014-07-01 17:04:47,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 4 to mira0435601-365 2014-07-01 17:04:47,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 6 to mira0435601-365 2014-07-01 17:04:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: finishing write tid 1 to mira0435601-407 2014-07-01 17:04:47,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: finishing write tid 3 to mira0435601-407 2014-07-01 17:04:47,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 1 to mira0435601-413 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 2 to mira0435601-413 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 3 to mira0435601-413 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 4 to mira0435601-413 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 6 to mira0435601-413 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 1 to mira0435601-391 2014-07-01 17:04:47,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: expect (ObjNum 1751 snap 0 seq_num 1751) 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 1 to mira0437250-428 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 2 to mira0437250-428 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 3 to mira0437250-428 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 5 to mira0437250-428 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 466 (ObjNum 489 snap 0 seq_num 489) dirty exists 2014-07-01 17:04:47,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 1 to mira0437250-177 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: finishing write tid 6 to mira0437250-428 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 704 (ObjNum 1158 snap 238 seq_num 1158) dirty exists 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 2 to mira0437250-177 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 3 to mira0437250-177 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: finishing write tid 7 to mira0437250-224 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 1320 (ObjNum 1159 snap 238 seq_num 1159) dirty exists 2014-07-01 17:04:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 5 to mira0437250-177 2014-07-01 17:04:49,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: finishing write tid 4 to mira0435601-407 2014-07-01 17:04:49,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 407 v 291 (ObjNum 2062 snap 0 seq_num 2062) dirty exists 2014-07-01 17:04:49,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 7 to mira0435601-365 2014-07-01 17:04:49,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 365 v 172 (ObjNum 2061 snap 0 seq_num 2061) dirty exists 2014-07-01 17:04:49,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: finishing write tid 7 to mira0435601-413 2014-07-01 17:04:49,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 321 (ObjNum 2065 snap 0 seq_num 2065) dirty exists 2014-07-01 17:04:49,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:3334: expect (ObjNum 384 snap 0 seq_num 384) 2014-07-01 17:04:51,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:2278: done (7 left) 2014-07-01 17:04:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2279: done (6 left) 2014-07-01 17:04:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2283: done (5 left) 2014-07-01 17:04:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2286: done (4 left) 2014-07-01 17:04:51,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2287: done (3 left) 2014-07-01 17:04:51,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2288: done (2 left) 2014-07-01 17:04:51,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: snap_create 2014-07-01 17:04:51,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: expect (ObjNum 1921 snap 0 seq_num 1921) 2014-07-01 17:04:51,211.211 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.2. 2014-07-01 17:04:51,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: finishing write tid 6 to mira0437250-177 2014-07-01 17:04:51,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 471 (ObjNum 1160 snap 238 seq_num 1160) dirty exists 2014-07-01 17:04:51,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: expect (ObjNum 918 snap 164 seq_num 918) 2014-07-01 17:04:52,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2274: done (2 left) 2014-07-01 17:04:52,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2284: done (1 left) 2014-07-01 17:04:52,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2289: done (0 left) 2014-07-01 17:04:52,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: read oid 429 snap -1 2014-07-01 17:04:52,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: write oid 430 current snap is 240 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: seq_num 1161 ranges {727616=501354,1862674=545874,2846743=735825,3927615=1} 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0437250-430 from 727616 to 1228970 tid 1 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0437250-430 from 1862674 to 2408548 tid 2 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0437250-430 from 2846743 to 3582568 tid 3 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: writing mira0437250-430 from 3927615 to 3927616 tid 4 2014-07-01 17:04:52,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: delete oid 332 current snap is 240 2014-07-01 17:04:53,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 1 to mira0437250-430 2014-07-01 17:04:53,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 2 to mira0437250-430 2014-07-01 17:04:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 3 to mira0437250-430 2014-07-01 17:04:53,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 4 to mira0437250-430 2014-07-01 17:04:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 6 to mira0437250-430 2014-07-01 17:04:53,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: finishing write tid 7 to mira0437250-430 2014-07-01 17:04:53,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 852 (ObjNum 1161 snap 240 seq_num 1161) dirty exists 2014-07-01 17:04:53,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:2291: done (2 left) 2014-07-01 17:04:53,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:2292: done (1 left) 2014-07-01 17:04:53,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: snap_remove snap 235 2014-07-01 17:04:53,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: expect (ObjNum 1041 snap 205 seq_num 1041) 2014-07-01 17:04:54,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2290: done (1 left) 2014-07-01 17:04:54,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2293: done (0 left) 2014-07-01 17:04:54,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: write oid 105 current snap is 240 2014-07-01 17:04:54,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: seq_num 1163 ranges {367675=1} 2014-07-01 17:04:54,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: writing mira0437250-105 from 367675 to 367676 tid 1 2014-07-01 17:04:54,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: snap_create 2014-07-01 17:04:54,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 1 to mira0435601-225 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 2 to mira0435601-225 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 3 to mira0435601-225 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 2 to mira0435601-391 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: finishing write tid 1 to mira0435601-261 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: finishing write tid 3 to mira0435601-261 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 3 to mira0435601-391 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: finishing write tid 4 to mira0435601-261 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 261 v 333 (ObjNum 2063 snap 0 seq_num 2063) dirty exists 2014-07-01 17:04:54,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 1 to mira0435601-201 2014-07-01 17:04:54,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 3 to mira0435601-201 2014-07-01 17:04:54,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 4 to mira0435601-391 2014-07-01 17:04:54,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3327: expect (ObjNum 1756 snap 0 seq_num 1756) 2014-07-01 17:04:55,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2295: done (1 left) 2014-07-01 17:04:55,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: snap_create 2014-07-01 17:04:56,221.221 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 3] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:04:56,221.221 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:04:56,221.221 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 17:04:56,221.221 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:04:56,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2296: done (1 left) 2014-07-01 17:04:56,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2297: snap_remove snap 223 2014-07-01 17:04:56,933.933 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:04:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 4 to mira0435601-201 2014-07-01 17:04:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 201 v 259 (ObjNum 2064 snap 0 seq_num 2064) dirty exists 2014-07-01 17:04:57,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: expect (ObjNum 1556 snap 0 seq_num 1556) 2014-07-01 17:04:57,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2297: done (1 left) 2014-07-01 17:04:57,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: snap_create 2014-07-01 17:04:58,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 1 to mira0437250-105 2014-07-01 17:04:58,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 3 to mira0437250-105 2014-07-01 17:04:59,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2298: done (1 left) 2014-07-01 17:04:59,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2299: snap_remove snap 237 2014-07-01 17:04:59,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: finishing write tid 4 to mira0437250-105 2014-07-01 17:04:59,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 1115 (ObjNum 1163 snap 240 seq_num 1163) dirty exists 2014-07-01 17:04:59,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2294: done (1 left) 2014-07-01 17:04:59,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2299: done (0 left) 2014-07-01 17:04:59,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: rollback oid 248 current snap is 243 2014-07-01 17:04:59,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 248 to 242 2014-07-01 17:04:59,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: snap_create 2014-07-01 17:05:00,194.194 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:05:00,195.195 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pg_num 54' 2014-07-01 17:05:00,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2301: done (1 left) 2014-07-01 17:05:00,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2302: read oid 76 snap -1 2014-07-01 17:05:00,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2303: snap_create 2014-07-01 17:05:01,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:2302: expect (ObjNum 75 snap 0 seq_num 75) 2014-07-01 17:05:01,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 6 to mira0435601-391 2014-07-01 17:05:01,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: finishing write tid 7 to mira0435601-391 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 323 (ObjNum 2066 snap 0 seq_num 2066) dirty exists 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 4 to mira0435601-225 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 6 to mira0435601-225 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: finishing write tid 7 to mira0435601-225 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 225 v 248 (ObjNum 2067 snap 0 seq_num 2067) dirty exists 2014-07-01 17:05:01,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3338: expect (ObjNum 1857 snap 0 seq_num 1857) 2014-07-01 17:05:01,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 248 v 803 (ObjNum 1102 snap 221 seq_num 1102) dirty exists 2014-07-01 17:05:01,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2300: done (2 left) 2014-07-01 17:05:01,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2302: done (1 left) 2014-07-01 17:05:01,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2303: done (0 left) 2014-07-01 17:05:01,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: delete oid 395 current snap is 245 2014-07-01 17:05:01,894.894 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pg_num to 54 2014-07-01 17:05:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2304: done (0 left) 2014-07-01 17:05:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: write oid 176 current snap is 245 2014-07-01 17:05:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: seq_num 1165 ranges {772663=425691,1946211=426452} 2014-07-01 17:05:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: writing mira0437250-176 from 772663 to 1198354 tid 1 2014-07-01 17:05:02,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: writing mira0437250-176 from 1946211 to 2372663 tid 2 2014-07-01 17:05:02,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: snap_create 2014-07-01 17:05:02,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:2306: done (1 left) 2014-07-01 17:05:02,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: write oid 476 current snap is 246 2014-07-01 17:05:02,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: seq_num 1166 ranges {468917=432567,1695415=373502} 2014-07-01 17:05:02,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: writing mira0437250-476 from 468917 to 901484 tid 1 2014-07-01 17:05:02,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: writing mira0437250-476 from 1695415 to 2068917 tid 2 2014-07-01 17:05:02,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: delete oid 39 current snap is 246 2014-07-01 17:05:03,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 1 to mira0435601-342 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 2 to mira0435601-342 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 4 to mira0435601-342 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 5 to mira0435601-342 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 234 (ObjNum 2068 snap 0 seq_num 2068) dirty exists 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: done (15 left) 2014-07-01 17:05:03,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: done (14 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: done (13 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: done (12 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3327: done (11 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: done (10 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: done (9 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: done (8 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: done (7 left) 2014-07-01 17:05:03,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: done (6 left) 2014-07-01 17:05:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: done (5 left) 2014-07-01 17:05:03,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:3334: done (4 left) 2014-07-01 17:05:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:3335: done (3 left) 2014-07-01 17:05:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:3336: done (2 left) 2014-07-01 17:05:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: done (1 left) 2014-07-01 17:05:03,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:3338: done (0 left) 2014-07-01 17:05:03,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3339: delete oid 97 current snap is 0 2014-07-01 17:05:03,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 1 to mira0437250-176 2014-07-01 17:05:03,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 2 to mira0437250-176 2014-07-01 17:05:03,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 4 to mira0437250-176 2014-07-01 17:05:03,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3339: done (0 left) 2014-07-01 17:05:03,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3340: delete oid 325 current snap is 0 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: finishing write tid 1 to mira0437250-476 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: finishing write tid 2 to mira0437250-476 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: finishing write tid 4 to mira0437250-476 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3340: done (0 left) 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 449 current snap is 0 2014-07-01 17:05:03,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: seq_num 2071 ranges {1830912=274432} 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: writing mira0435601-449 from 1830912 to 2105344 tid 1 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3342: read oid 31 snap -1 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3343: read oid 319 snap -1 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: read oid 209 snap -1 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 452 current snap is 0 2014-07-01 17:05:04,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: seq_num 2072 ranges {0=147456} 2014-07-01 17:05:04,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: writing mira0435601-452 from 0 to 147456 tid 1 2014-07-01 17:05:04,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 1 current snap is 0 2014-07-01 17:05:04,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: seq_num 2073 ranges {1417216=20480} 2014-07-01 17:05:04,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: writing mira0435601-1 from 1417216 to 1437696 tid 1 2014-07-01 17:05:04,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3347: delete oid 321 current snap is 0 2014-07-01 17:05:04,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: finishing write tid 5 to mira0437250-476 2014-07-01 17:05:04,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 1149 (ObjNum 1166 snap 246 seq_num 1166) dirty exists 2014-07-01 17:05:04,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: finishing write tid 5 to mira0437250-176 2014-07-01 17:05:04,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 1220 (ObjNum 1165 snap 245 seq_num 1165) dirty exists 2014-07-01 17:05:04,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2305: done (2 left) 2014-07-01 17:05:04,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2307: done (1 left) 2014-07-01 17:05:04,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2308: done (0 left) 2014-07-01 17:05:04,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: write oid 37 current snap is 246 2014-07-01 17:05:04,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: seq_num 1168 ranges {72880=1} 2014-07-01 17:05:04,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: writing mira0437250-37 from 72880 to 72881 tid 1 2014-07-01 17:05:05,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: write oid 166 current snap is 246 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: seq_num 1169 ranges {569134=500428,1565619=645355,2935522=665862,3769133=1} 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: writing mira0437250-166 from 569134 to 1069562 tid 1 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: writing mira0437250-166 from 1565619 to 2210974 tid 2 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: writing mira0437250-166 from 2935522 to 3601384 tid 3 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: writing mira0437250-166 from 3769133 to 3769134 tid 4 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: read oid 194 snap 231 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: read oid 189 snap -1 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2313: snap_remove snap 234 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: expect (ObjNum 943 snap 173 seq_num 943) 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: finishing write tid 1 to mira0435601-1 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: finishing write tid 3 to mira0435601-1 2014-07-01 17:05:05,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3343: expect (ObjNum 1879 snap 0 seq_num 1879) 2014-07-01 17:05:05,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: expect (ObjNum 193 snap 0 seq_num 193) 2014-07-01 17:05:06,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: finishing write tid 1 to mira0437250-37 2014-07-01 17:05:06,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: finishing write tid 3 to mira0437250-37 2014-07-01 17:05:06,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: finishing write tid 4 to mira0437250-37 2014-07-01 17:05:06,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 926 (ObjNum 1168 snap 246 seq_num 1168) dirty exists 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2309: done (4 left) 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2311: done (3 left) 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2312: done (2 left) 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2313: done (1 left) 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: write oid 498 current snap is 246 2014-07-01 17:05:06,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: seq_num 1170 ranges {512939=1} 2014-07-01 17:05:06,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: writing mira0437250-498 from 512939 to 512940 tid 1 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: write oid 302 current snap is 246 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: seq_num 1171 ranges {209193=1} 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: writing mira0437250-302 from 209193 to 209194 tid 1 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: write oid 19 current snap is 246 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: seq_num 1172 ranges {553000=400000} 2014-07-01 17:05:06,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: writing mira0437250-19 from 553000 to 953000 tid 1 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: write oid 300 current snap is 246 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: seq_num 1173 ranges {614181=423742,1799556=414625} 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: writing mira0437250-300 from 614181 to 1037923 tid 1 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: writing mira0437250-300 from 1799556 to 2214181 tid 2 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: rollback oid 63 current snap is 246 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 63 to 194 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: write oid 103 current snap is 246 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: seq_num 1174 ranges {710435=546970,1910434=1} 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: writing mira0437250-103 from 710435 to 1257405 tid 1 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: writing mira0437250-103 from 1910434 to 1910435 tid 2 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: snap_create 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2320: done (7 left) 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2321: delete oid 151 current snap is 247 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2321: done (7 left) 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2322: read oid 24 snap 229 2014-07-01 17:05:06,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2323: delete oid 372 current snap is 247 2014-07-01 17:05:06,906.906 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 3] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:06,906.906 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:06,906.906 INFO:teuthology.task.thrashosds.thrasher:Growing pool rbd 2014-07-01 17:05:06,907.907 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:05:07,536.536 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:05:08,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 1 to mira0437250-302 2014-07-01 17:05:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 3 to mira0437250-302 2014-07-01 17:05:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 914 (ObjNum 62 snap 0 seq_num 62) dirty exists 2014-07-01 17:05:08,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: finishing write tid 1 to mira0437250-103 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: finishing write tid 2 to mira0437250-103 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: finishing write tid 4 to mira0437250-103 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: finishing write tid 4 to mira0437250-302 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 302 v 912 (ObjNum 1171 snap 246 seq_num 1171) dirty exists 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: finishing write tid 5 to mira0437250-103 2014-07-01 17:05:08,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 103 v 932 (ObjNum 1174 snap 246 seq_num 1174) dirty exists 2014-07-01 17:05:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 1 to mira0437250-166 2014-07-01 17:05:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 2 to mira0437250-166 2014-07-01 17:05:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 3 to mira0437250-166 2014-07-01 17:05:08,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 4 to mira0437250-166 2014-07-01 17:05:08,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 6 to mira0437250-166 2014-07-01 17:05:08,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 1 to mira0437250-19 2014-07-01 17:05:08,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2322: expect (ObjNum 1003 snap 194 seq_num 1003) 2014-07-01 17:05:09,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: finishing write tid 4 to mira0435601-1 2014-07-01 17:05:09,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 206 (ObjNum 2073 snap 0 seq_num 2073) dirty exists 2014-07-01 17:05:09,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: finishing write tid 1 to mira0435601-449 2014-07-01 17:05:09,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: finishing write tid 3 to mira0435601-449 2014-07-01 17:05:09,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3342: expect (ObjNum 1947 snap 0 seq_num 1947) 2014-07-01 17:05:09,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 3 to mira0437250-19 2014-07-01 17:05:09,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: finishing write tid 7 to mira0437250-166 2014-07-01 17:05:09,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 532 (ObjNum 1169 snap 246 seq_num 1169) dirty exists 2014-07-01 17:05:09,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: finishing write tid 1 to mira0437250-498 2014-07-01 17:05:09,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: finishing write tid 3 to mira0437250-498 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: finishing write tid 4 to mira0437250-498 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 784 (ObjNum 1170 snap 246 seq_num 1170) dirty exists 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: finishing write tid 1 to mira0437250-300 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2310: done (8 left) 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2314: done (7 left) 2014-07-01 17:05:09,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:2315: done (6 left) 2014-07-01 17:05:09,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2318: done (5 left) 2014-07-01 17:05:09,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2319: done (4 left) 2014-07-01 17:05:09,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2322: done (3 left) 2014-07-01 17:05:09,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2323: done (2 left) 2014-07-01 17:05:09,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2324: read oid 348 snap -1 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: finishing write tid 4 to mira0437250-19 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 19 v 775 (ObjNum 1172 snap 246 seq_num 1172) dirty exists 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: finishing write tid 2 to mira0437250-300 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: finishing write tid 4 to mira0437250-300 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: finishing write tid 5 to mira0437250-300 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 1507 (ObjNum 1173 snap 246 seq_num 1173) dirty exists 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2316: done (2 left) 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2317: done (1 left) 2014-07-01 17:05:09,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: snap_remove snap 243 2014-07-01 17:05:09,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2324: expect (ObjNum 802 snap 129 seq_num 802) 2014-07-01 17:05:10,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2324: done (1 left) 2014-07-01 17:05:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2325: done (0 left) 2014-07-01 17:05:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: rollback oid 488 current snap is 247 2014-07-01 17:05:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 488 to 194 2014-07-01 17:05:10,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: read oid 351 snap -1 2014-07-01 17:05:10,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: write oid 213 current snap is 247 2014-07-01 17:05:10,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: seq_num 1177 ranges {410652=498479,1667540=592389,3022253=480570,3610651=1} 2014-07-01 17:05:10,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0437250-213 from 410652 to 909131 tid 1 2014-07-01 17:05:10,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0437250-213 from 1667540 to 2259929 tid 2 2014-07-01 17:05:10,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0437250-213 from 3022253 to 3502823 tid 3 2014-07-01 17:05:10,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: writing mira0437250-213 from 3610651 to 3610652 tid 4 2014-07-01 17:05:10,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: snap_create 2014-07-01 17:05:10,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 796 (ObjNum 649 snap 69 seq_num 649) dirty exists 2014-07-01 17:05:11,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2326: done (3 left) 2014-07-01 17:05:11,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2329: done (2 left) 2014-07-01 17:05:11,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2330: snap_remove snap 230 2014-07-01 17:05:11,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: expect (ObjNum 1047 snap 206 seq_num 1047) 2014-07-01 17:05:11,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 1 to mira0437250-213 2014-07-01 17:05:11,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 2 to mira0437250-213 2014-07-01 17:05:11,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 3 to mira0437250-213 2014-07-01 17:05:11,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 4 to mira0437250-213 2014-07-01 17:05:12,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2327: done (2 left) 2014-07-01 17:05:12,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2330: done (1 left) 2014-07-01 17:05:12,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: snap_create 2014-07-01 17:05:12,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 6 to mira0437250-213 2014-07-01 17:05:12,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: finishing write tid 7 to mira0437250-213 2014-07-01 17:05:12,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 1082 (ObjNum 1177 snap 247 seq_num 1177) dirty exists 2014-07-01 17:05:13,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2328: done (1 left) 2014-07-01 17:05:13,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2331: done (0 left) 2014-07-01 17:05:13,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2332: snap_create 2014-07-01 17:05:13,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: finishing write tid 4 to mira0435601-449 2014-07-01 17:05:13,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 274 (ObjNum 2071 snap 0 seq_num 2071) dirty exists 2014-07-01 17:05:13,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: expect (ObjNum 1397 snap 0 seq_num 1397) 2014-07-01 17:05:14,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:2332: done (0 left) 2014-07-01 17:05:14,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: delete oid 179 current snap is 250 2014-07-01 17:05:14,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: finishing write tid 1 to mira0435601-452 2014-07-01 17:05:14,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: finishing write tid 3 to mira0435601-452 2014-07-01 17:05:14,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: finishing write tid 4 to mira0435601-452 2014-07-01 17:05:14,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 310 (ObjNum 2072 snap 0 seq_num 2072) dirty exists 2014-07-01 17:05:14,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: done (6 left) 2014-07-01 17:05:14,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3342: done (5 left) 2014-07-01 17:05:14,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3343: done (4 left) 2014-07-01 17:05:14,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: done (3 left) 2014-07-01 17:05:14,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: done (2 left) 2014-07-01 17:05:14,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: done (1 left) 2014-07-01 17:05:14,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3347: done (0 left) 2014-07-01 17:05:14,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3348: delete oid 137 current snap is 0 2014-07-01 17:05:15,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:2333: done (0 left) 2014-07-01 17:05:15,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: snap_remove snap 211 2014-07-01 17:05:15,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3348: done (0 left) 2014-07-01 17:05:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 379 current snap is 0 2014-07-01 17:05:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: seq_num 2076 ranges {4096000=643072,4739072=749568,5488640=454656} 2014-07-01 17:05:15,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: writing mira0435601-379 from 4096000 to 4739072 tid 1 2014-07-01 17:05:15,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: writing mira0435601-379 from 4739072 to 5488640 tid 2 2014-07-01 17:05:15,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: writing mira0435601-379 from 5488640 to 5943296 tid 3 2014-07-01 17:05:15,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3350: read oid 296 snap -1 2014-07-01 17:05:15,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3351: read oid 500 snap -1 2014-07-01 17:05:15,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: read oid 244 snap -1 2014-07-01 17:05:15,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: read oid 197 snap -1 2014-07-01 17:05:15,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3354: read oid 320 snap -1 2014-07-01 17:05:15,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 250 current snap is 0 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: seq_num 2077 ranges {299008=667648} 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: writing mira0435601-250 from 299008 to 966656 tid 1 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: seq_num 2078 ranges {11894784=540672} 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: writing mira0435601-337 from 11894784 to 12435456 tid 1 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2334: done (0 left) 2014-07-01 17:05:15,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: write oid 350 current snap is 250 2014-07-01 17:05:15,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: seq_num 1179 ranges {699288=400000} 2014-07-01 17:05:15,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: writing mira0437250-350 from 699288 to 1099288 tid 1 2014-07-01 17:05:15,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:2336: snap_remove snap 236 2014-07-01 17:05:15,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: read oid 421 snap -1 2014-07-01 17:05:15,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3351: expect (ObjNum 1997 snap 0 seq_num 1997) 2014-07-01 17:05:16,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2336: done (1 left) 2014-07-01 17:05:16,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: write oid 469 current snap is 250 2014-07-01 17:05:16,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: seq_num 1180 ranges {794516=1} 2014-07-01 17:05:16,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: writing mira0437250-469 from 794516 to 794517 tid 1 2014-07-01 17:05:16,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: write oid 354 current snap is 250 2014-07-01 17:05:16,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: seq_num 1181 ranges {739347=573216,1539346=1} 2014-07-01 17:05:16,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: writing mira0437250-354 from 739347 to 1312563 tid 1 2014-07-01 17:05:16,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: writing mira0437250-354 from 1539346 to 1539347 tid 2 2014-07-01 17:05:16,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: read oid 16 snap -1 2014-07-01 17:05:16,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: write oid 202 current snap is 250 2014-07-01 17:05:16,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: seq_num 1182 ranges {400529=697469,1617658=481285,2679005=121524} 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0437250-202 from 400529 to 1097998 tid 1 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0437250-202 from 1617658 to 2098943 tid 2 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: writing mira0437250-202 from 2679005 to 2800529 tid 3 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: read oid 243 snap -1 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: finishing write tid 1 to mira0437250-350 2014-07-01 17:05:16,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: expect (ObjNum 1130 snap 230 seq_num 1130) 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: finishing write tid 3 to mira0437250-350 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: finishing write tid 1 to mira0437250-354 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: finishing write tid 2 to mira0437250-354 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: finishing write tid 4 to mira0437250-354 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: finishing write tid 5 to mira0437250-354 2014-07-01 17:05:16,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 354 v 1236 (ObjNum 1181 snap 250 seq_num 1181) dirty exists 2014-07-01 17:05:16,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: expect (ObjNum 585 snap 35 seq_num 585) 2014-07-01 17:05:16,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3350: expect (ObjNum 1808 snap 0 seq_num 1808) 2014-07-01 17:05:16,736.736 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 3] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:16,736.736 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:16,737.737 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 17:05:16,737.737 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:05:16,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: expect (ObjNum 1605 snap 0 seq_num 1605) 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: finishing write tid 4 to mira0437250-350 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 1087 (ObjNum 1179 snap 250 seq_num 1179) dirty exists 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 1 to mira0437250-202 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 2 to mira0437250-202 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 3 to mira0437250-202 2014-07-01 17:05:17,073.073 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 5 to mira0437250-202 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: finishing write tid 6 to mira0437250-202 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 786 (ObjNum 1182 snap 250 seq_num 1182) dirty exists 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: finishing write tid 1 to mira0437250-469 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2335: done (5 left) 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2338: done (4 left) 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2339: done (3 left) 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2340: done (2 left) 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2341: done (1 left) 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: rollback oid 125 current snap is 250 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 125 to 194 2014-07-01 17:05:17,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: finishing write tid 3 to mira0437250-469 2014-07-01 17:05:17,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: finishing write tid 4 to mira0437250-469 2014-07-01 17:05:17,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 1102 (ObjNum 1180 snap 250 seq_num 1180) dirty exists 2014-07-01 17:05:17,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:2337: done (1 left) 2014-07-01 17:05:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: write oid 150 current snap is 250 2014-07-01 17:05:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: seq_num 1183 ranges {494735=401497,1425870=551480,2457958=36777} 2014-07-01 17:05:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0437250-150 from 494735 to 896232 tid 1 2014-07-01 17:05:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0437250-150 from 1425870 to 1977350 tid 2 2014-07-01 17:05:17,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: writing mira0437250-150 from 2457958 to 2494735 tid 3 2014-07-01 17:05:17,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2344: snap_remove snap 248 2014-07-01 17:05:17,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 560 (ObjNum 506 snap 3 seq_num 506) dirty exists 2014-07-01 17:05:17,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 1 to mira0437250-150 2014-07-01 17:05:17,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 2 to mira0437250-150 2014-07-01 17:05:17,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 3 to mira0437250-150 2014-07-01 17:05:17,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 5 to mira0437250-150 2014-07-01 17:05:17,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: finishing write tid 6 to mira0437250-150 2014-07-01 17:05:17,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 150 v 723 (ObjNum 1183 snap 250 seq_num 1183) dirty exists 2014-07-01 17:05:17,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:2342: done (2 left) 2014-07-01 17:05:17,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:2343: done (1 left) 2014-07-01 17:05:17,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:2344: done (0 left) 2014-07-01 17:05:17,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:2345: delete oid 41 current snap is 250 2014-07-01 17:05:17,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:2345: done (0 left) 2014-07-01 17:05:17,409.409 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: snap_create 2014-07-01 17:05:17,515.515 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:05:18,361.361 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: finishing write tid 1 to mira0435601-379 2014-07-01 17:05:18,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: finishing write tid 2 to mira0435601-379 2014-07-01 17:05:18,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: finishing write tid 3 to mira0435601-379 2014-07-01 17:05:18,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: finishing write tid 5 to mira0435601-379 2014-07-01 17:05:18,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: finishing write tid 6 to mira0435601-379 2014-07-01 17:05:18,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 260 (ObjNum 2076 snap 0 seq_num 2076) dirty exists 2014-07-01 17:05:18,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3354: expect (ObjNum 952 snap 0 seq_num 952) 2014-07-01 17:05:18,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:2346: done (0 left) 2014-07-01 17:05:18,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: read oid 369 snap -1 2014-07-01 17:05:18,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2348: rollback oid 314 current snap is 251 2014-07-01 17:05:18,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 314 to 197 2014-07-01 17:05:18,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: read oid 64 snap -1 2014-07-01 17:05:18,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: write oid 214 current snap is 251 2014-07-01 17:05:18,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: seq_num 1185 ranges {783370=496530,2018038=691870,3474937=508433} 2014-07-01 17:05:18,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: writing mira0437250-214 from 783370 to 1279900 tid 1 2014-07-01 17:05:18,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: writing mira0437250-214 from 2018038 to 2709908 tid 2 2014-07-01 17:05:18,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: writing mira0437250-214 from 3474937 to 3983370 tid 3 2014-07-01 17:05:18,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: write oid 203 current snap is 251 2014-07-01 17:05:18,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: seq_num 1186 ranges {195975=1} 2014-07-01 17:05:18,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: writing mira0437250-203 from 195975 to 195976 tid 1 2014-07-01 17:05:18,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: write oid 429 current snap is 251 2014-07-01 17:05:18,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: seq_num 1187 ranges {540806=400000} 2014-07-01 17:05:18,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: writing mira0437250-429 from 540806 to 940806 tid 1 2014-07-01 17:05:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: read oid 315 snap -1 2014-07-01 17:05:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 1240 (ObjNum 313 snap 0 seq_num 313) dirty exists 2014-07-01 17:05:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: expect (ObjNum 368 snap 0 seq_num 368) 2014-07-01 17:05:19,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: finishing write tid 1 to mira0437250-214 2014-07-01 17:05:19,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: expect (ObjNum 1060 snap 211 seq_num 1060) 2014-07-01 17:05:19,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: expect (ObjNum 1982 snap 0 seq_num 1982) 2014-07-01 17:05:20,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: finishing write tid 2 to mira0437250-214 2014-07-01 17:05:20,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: finishing write tid 3 to mira0437250-214 2014-07-01 17:05:20,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: finishing write tid 5 to mira0437250-214 2014-07-01 17:05:20,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: finishing write tid 1 to mira0437250-203 2014-07-01 17:05:20,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: finishing write tid 3 to mira0437250-203 2014-07-01 17:05:20,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 1 to mira0437250-429 2014-07-01 17:05:20,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 3 to mira0437250-429 2014-07-01 17:05:20,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: finishing write tid 4 to mira0437250-203 2014-07-01 17:05:20,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 867 (ObjNum 1186 snap 251 seq_num 1186) dirty exists 2014-07-01 17:05:20,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: finishing write tid 6 to mira0437250-214 2014-07-01 17:05:20,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 214 v 795 (ObjNum 1185 snap 251 seq_num 1185) dirty exists 2014-07-01 17:05:20,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: finishing write tid 4 to mira0437250-429 2014-07-01 17:05:20,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 1009 (ObjNum 1187 snap 251 seq_num 1187) dirty exists 2014-07-01 17:05:20,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2347: done (6 left) 2014-07-01 17:05:20,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2348: done (5 left) 2014-07-01 17:05:20,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2349: done (4 left) 2014-07-01 17:05:20,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2350: done (3 left) 2014-07-01 17:05:20,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2351: done (2 left) 2014-07-01 17:05:20,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2352: done (1 left) 2014-07-01 17:05:20,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2353: done (0 left) 2014-07-01 17:05:20,381.381 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: write oid 486 current snap is 251 2014-07-01 17:05:20,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: seq_num 1188 ranges {484611=599463,1525362=159249} 2014-07-01 17:05:20,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: writing mira0437250-486 from 484611 to 1084074 tid 1 2014-07-01 17:05:20,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: writing mira0437250-486 from 1525362 to 1684611 tid 2 2014-07-01 17:05:20,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2355: snap_create 2014-07-01 17:05:20,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: finishing write tid 1 to mira0437250-486 2014-07-01 17:05:20,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: finishing write tid 2 to mira0437250-486 2014-07-01 17:05:20,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: finishing write tid 4 to mira0437250-486 2014-07-01 17:05:20,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: finishing write tid 5 to mira0437250-486 2014-07-01 17:05:20,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 812 (ObjNum 1188 snap 251 seq_num 1188) dirty exists 2014-07-01 17:05:20,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2354: done (1 left) 2014-07-01 17:05:20,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2355: done (0 left) 2014-07-01 17:05:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: write oid 384 current snap is 252 2014-07-01 17:05:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: seq_num 1189 ranges {580865=572291,1380864=1} 2014-07-01 17:05:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: writing mira0437250-384 from 580865 to 1153156 tid 1 2014-07-01 17:05:20,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: writing mira0437250-384 from 1380864 to 1380865 tid 2 2014-07-01 17:05:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: read oid 58 snap -1 2014-07-01 17:05:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2358: snap_remove snap 227 2014-07-01 17:05:20,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: expect (ObjNum 57 snap 0 seq_num 57) 2014-07-01 17:05:21,129.129 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 56' 2014-07-01 17:05:22,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 1 to mira0437250-384 2014-07-01 17:05:22,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 2 to mira0437250-384 2014-07-01 17:05:22,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 4 to mira0437250-384 2014-07-01 17:05:22,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2357: done (2 left) 2014-07-01 17:05:22,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2358: done (1 left) 2014-07-01 17:05:22,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2359: rollback oid 381 current snap is 252 2014-07-01 17:05:22,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 381 to 251 2014-07-01 17:05:22,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: read oid 17 snap 244 2014-07-01 17:05:22,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: delete oid 78 current snap is 252 2014-07-01 17:05:22,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2361: done (3 left) 2014-07-01 17:05:22,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: snap_remove snap 232 2014-07-01 17:05:22,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:2362: done (3 left) 2014-07-01 17:05:22,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:2363: rollback oid 431 current snap is 252 2014-07-01 17:05:22,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 431 to 194 2014-07-01 17:05:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: write oid 184 current snap is 252 2014-07-01 17:05:22,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: seq_num 1191 ranges {585853=550972,1680239=499537,2663089=605205,3385852=1} 2014-07-01 17:05:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0437250-184 from 585853 to 1136825 tid 1 2014-07-01 17:05:22,539.539 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0437250-184 from 1680239 to 2179776 tid 2 2014-07-01 17:05:22,541.541 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 56 2014-07-01 17:05:22,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0437250-184 from 2663089 to 3268294 tid 3 2014-07-01 17:05:22,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: writing mira0437250-184 from 3385852 to 3385853 tid 4 2014-07-01 17:05:22,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: read oid 114 snap 233 2014-07-01 17:05:22,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 1 to mira0437250-184 2014-07-01 17:05:22,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: rollback oid 459 current snap is 252 2014-07-01 17:05:22,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 459 to 246 2014-07-01 17:05:22,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2367: delete oid 464 current snap is 252 2014-07-01 17:05:22,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 2 to mira0437250-184 2014-07-01 17:05:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 459 v 1246 (ObjNum 458 snap 0 seq_num 458) dirty exists 2014-07-01 17:05:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: finishing write tid 5 to mira0437250-384 2014-07-01 17:05:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 1393 (ObjNum 1189 snap 252 seq_num 1189) dirty exists 2014-07-01 17:05:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 0 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:05:22,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 3 to mira0437250-184 2014-07-01 17:05:22,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 4 to mira0437250-184 2014-07-01 17:05:22,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 6 to mira0437250-184 2014-07-01 17:05:22,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: finishing write tid 7 to mira0437250-184 2014-07-01 17:05:22,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 830 (ObjNum 1191 snap 252 seq_num 1191) dirty exists 2014-07-01 17:05:22,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: expect (ObjNum 1131 snap 230 seq_num 1131) 2014-07-01 17:05:23,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: expect (ObjNum 1152 snap 237 seq_num 1152) 2014-07-01 17:05:23,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 431 v 1154 (ObjNum -1726349310 snap -2138581494 seq_num 0) dirty dne 2014-07-01 17:05:23,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2356: done (7 left) 2014-07-01 17:05:23,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2359: done (6 left) 2014-07-01 17:05:23,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:2360: done (5 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2363: done (4 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2364: done (3 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2365: done (2 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2366: done (1 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2367: done (0 left) 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: read oid 32 snap -1 2014-07-01 17:05:23,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:2369: delete oid 492 current snap is 252 2014-07-01 17:05:23,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2368: done (1 left) 2014-07-01 17:05:23,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2369: done (0 left) 2014-07-01 17:05:23,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: rollback oid 251 current snap is 252 2014-07-01 17:05:23,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 251 to 241 2014-07-01 17:05:23,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2371: read oid 117 snap -1 2014-07-01 17:05:23,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2372: snap_create 2014-07-01 17:05:23,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 709 (ObjNum 250 snap 0 seq_num 250) dirty exists 2014-07-01 17:05:23,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: expect (ObjNum 1998 snap 0 seq_num 1998) 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2370: done (2 left) 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2371: done (1 left) 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2372: done (0 left) 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2373: read oid 20 snap 218 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2374: read oid 155 snap -1 2014-07-01 17:05:23,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: snap_create 2014-07-01 17:05:23,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2373: expect (ObjNum 702 snap 92 seq_num 702) 2014-07-01 17:05:24,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2373: done (2 left) 2014-07-01 17:05:24,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2374: done (1 left) 2014-07-01 17:05:24,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:2375: done (0 left) 2014-07-01 17:05:24,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: rollback oid 293 current snap is 254 2014-07-01 17:05:24,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 293 to 229 2014-07-01 17:05:24,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: rollback oid 147 current snap is 254 2014-07-01 17:05:24,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 147 to 252 2014-07-01 17:05:24,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: rollback oid 171 current snap is 254 2014-07-01 17:05:24,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 171 to 249 2014-07-01 17:05:24,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: delete oid 458 current snap is 254 2014-07-01 17:05:25,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 694 (ObjNum 170 snap 0 seq_num 170) dirty exists 2014-07-01 17:05:25,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2378: done (3 left) 2014-07-01 17:05:25,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:2379: done (2 left) 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: write oid 135 current snap is 254 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: seq_num 1195 ranges {781299=1} 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: writing mira0437250-135 from 781299 to 781300 tid 1 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2381: delete oid 99 current snap is 254 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 874 (ObjNum 884 snap 150 seq_num 884) dirty exists 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 1156 (ObjNum 1067 snap 213 seq_num 1067) dirty exists 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: finishing write tid 1 to mira0437250-135 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: finishing write tid 3 to mira0437250-135 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2376: done (3 left) 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2377: done (2 left) 2014-07-01 17:05:25,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:2381: done (1 left) 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: write oid 234 current snap is 254 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: seq_num 1197 ranges {422383=571366,1222382=1} 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: writing mira0437250-234 from 422383 to 993749 tid 1 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: writing mira0437250-234 from 1222382 to 1222383 tid 2 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: read oid 251 snap -1 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: finishing write tid 4 to mira0437250-135 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 135 v 566 (ObjNum 1195 snap 254 seq_num 1195) dirty exists 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2380: done (2 left) 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: write oid 212 current snap is 254 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: seq_num 1198 ranges {766189=694594,1966188=1} 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: writing mira0437250-212 from 766189 to 1460783 tid 1 2014-07-01 17:05:25,145.145 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: writing mira0437250-212 from 1966188 to 1966189 tid 2 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: write oid 120 current snap is 254 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: seq_num 1199 ranges {711018=550047,1666832=597995,2711017=1} 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: writing mira0437250-120 from 711018 to 1261065 tid 1 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: writing mira0437250-120 from 1666832 to 2264827 tid 2 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: writing mira0437250-120 from 2711017 to 2711018 tid 3 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: write oid 253 current snap is 254 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: seq_num 1200 ranges {521576=674299,1764068=419614,2921575=1} 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: writing mira0437250-253 from 521576 to 1195875 tid 1 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: writing mira0437250-253 from 1764068 to 2183682 tid 2 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: writing mira0437250-253 from 2921575 to 2921576 tid 3 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: write oid 454 current snap is 254 2014-07-01 17:05:25,146.146 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: seq_num 1201 ranges {466406=646103,1726776=489809,2905742=440022,3666405=1} 2014-07-01 17:05:25,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0437250-454 from 466406 to 1112509 tid 1 2014-07-01 17:05:25,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0437250-454 from 1726776 to 2216585 tid 2 2014-07-01 17:05:25,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0437250-454 from 2905742 to 3345764 tid 3 2014-07-01 17:05:25,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: writing mira0437250-454 from 3666405 to 3666406 tid 4 2014-07-01 17:05:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2388: snap_remove snap 231 2014-07-01 17:05:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: finishing write tid 1 to mira0437250-234 2014-07-01 17:05:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: finishing write tid 2 to mira0437250-234 2014-07-01 17:05:25,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: finishing write tid 4 to mira0437250-234 2014-07-01 17:05:25,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: finishing write tid 5 to mira0437250-234 2014-07-01 17:05:25,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 779 (ObjNum 1197 snap 254 seq_num 1197) dirty exists 2014-07-01 17:05:25,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: expect (ObjNum 250 snap 0 seq_num 250) 2014-07-01 17:05:25,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: finishing write tid 1 to mira0437250-253 2014-07-01 17:05:25,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 1 to mira0437250-212 2014-07-01 17:05:25,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 2 to mira0437250-212 2014-07-01 17:05:25,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 4 to mira0437250-212 2014-07-01 17:05:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: finishing write tid 1 to mira0437250-120 2014-07-01 17:05:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: finishing write tid 2 to mira0437250-120 2014-07-01 17:05:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: finishing write tid 3 to mira0437250-120 2014-07-01 17:05:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: finishing write tid 5 to mira0437250-120 2014-07-01 17:05:25,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: finishing write tid 2 to mira0437250-253 2014-07-01 17:05:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: finishing write tid 3 to mira0437250-253 2014-07-01 17:05:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: finishing write tid 5 to mira0437250-253 2014-07-01 17:05:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 1 to mira0437250-454 2014-07-01 17:05:25,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 2 to mira0437250-454 2014-07-01 17:05:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2382: done (6 left) 2014-07-01 17:05:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2383: done (5 left) 2014-07-01 17:05:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2388: done (4 left) 2014-07-01 17:05:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: read oid 210 snap -1 2014-07-01 17:05:25,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 3 to mira0437250-454 2014-07-01 17:05:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 4 to mira0437250-454 2014-07-01 17:05:25,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 6 to mira0437250-454 2014-07-01 17:05:25,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: finishing write tid 5 to mira0437250-212 2014-07-01 17:05:25,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 953 (ObjNum 1198 snap 254 seq_num 1198) dirty exists 2014-07-01 17:05:25,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: finishing write tid 6 to mira0437250-120 2014-07-01 17:05:25,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 828 (ObjNum 1199 snap 254 seq_num 1199) dirty exists 2014-07-01 17:05:25,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: finishing write tid 6 to mira0437250-253 2014-07-01 17:05:25,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 1180 (ObjNum 1200 snap 254 seq_num 1200) dirty exists 2014-07-01 17:05:25,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: finishing write tid 7 to mira0437250-454 2014-07-01 17:05:25,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 910 (ObjNum 1201 snap 254 seq_num 1201) dirty exists 2014-07-01 17:05:25,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:2384: done (4 left) 2014-07-01 17:05:25,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2385: done (3 left) 2014-07-01 17:05:25,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2386: done (2 left) 2014-07-01 17:05:25,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2387: done (1 left) 2014-07-01 17:05:25,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: snap_remove snap 218 2014-07-01 17:05:25,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: expect (ObjNum 987 snap 187 seq_num 987) 2014-07-01 17:05:26,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:2389: done (1 left) 2014-07-01 17:05:26,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:2390: done (0 left) 2014-07-01 17:05:26,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: read oid 214 snap -1 2014-07-01 17:05:26,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: write oid 73 current snap is 254 2014-07-01 17:05:26,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: seq_num 1202 ranges {411235=1} 2014-07-01 17:05:26,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: writing mira0437250-73 from 411235 to 411236 tid 1 2014-07-01 17:05:26,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: read oid 202 snap -1 2014-07-01 17:05:26,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2394: rollback oid 324 current snap is 254 2014-07-01 17:05:26,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 324 to 156 2014-07-01 17:05:26,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: write oid 409 current snap is 254 2014-07-01 17:05:26,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: seq_num 1203 ranges {106464=1} 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: writing mira0437250-409 from 106464 to 106465 tid 1 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: rollback oid 34 current snap is 254 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 34 to 240 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: write oid 184 current snap is 254 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: seq_num 1204 ranges {567647=749036,1367646=1} 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: writing mira0437250-184 from 567647 to 1316683 tid 1 2014-07-01 17:05:26,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: writing mira0437250-184 from 1367646 to 1367647 tid 2 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: write oid 129 current snap is 254 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: seq_num 1205 ranges {511453=569417,1809289=302164} 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: writing mira0437250-129 from 511453 to 1080870 tid 1 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: writing mira0437250-129 from 1809289 to 2111453 tid 2 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: snap_remove snap 252 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 0 (ObjNum 29 snap 0 seq_num 0) dirty dne 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: finishing write tid 1 to mira0437250-73 2014-07-01 17:05:26,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: finishing write tid 3 to mira0437250-73 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: finishing write tid 4 to mira0437250-73 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 515 (ObjNum 1202 snap 254 seq_num 1202) dirty exists 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 1 to mira0437250-409 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 3 to mira0437250-409 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 34 v 848 (ObjNum 825 snap 132 seq_num 825) dirty exists 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: finishing write tid 4 to mira0437250-409 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 787 (ObjNum 1203 snap 254 seq_num 1203) dirty exists 2014-07-01 17:05:26,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 1 to mira0437250-184 2014-07-01 17:05:26,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 2 to mira0437250-184 2014-07-01 17:05:26,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 4 to mira0437250-184 2014-07-01 17:05:26,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: finishing write tid 5 to mira0437250-184 2014-07-01 17:05:26,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 850 (ObjNum 1204 snap 254 seq_num 1204) dirty exists 2014-07-01 17:05:26,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: expect (ObjNum 1185 snap 251 seq_num 1185) 2014-07-01 17:05:27,551.551 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 3] out_osds: [2] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:27,551.551 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:27,552.552 INFO:teuthology.task.thrashosds.thrasher:Removing osd 3, in_osds are: [5, 4, 0, 1, 3] 2014-07-01 17:05:27,553.553 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 3' 2014-07-01 17:05:27,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: finishing write tid 1 to mira0435601-250 2014-07-01 17:05:27,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: finishing write tid 3 to mira0435601-250 2014-07-01 17:05:27,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: finishing write tid 4 to mira0435601-250 2014-07-01 17:05:27,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 163 (ObjNum 2077 snap 0 seq_num 2077) dirty exists 2014-07-01 17:05:27,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: done (8 left) 2014-07-01 17:05:27,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3350: done (7 left) 2014-07-01 17:05:27,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3351: done (6 left) 2014-07-01 17:05:27,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: done (5 left) 2014-07-01 17:05:27,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: done (4 left) 2014-07-01 17:05:27,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3354: done (3 left) 2014-07-01 17:05:27,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: done (2 left) 2014-07-01 17:05:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: done (1 left) 2014-07-01 17:05:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 223 current snap is 0 2014-07-01 17:05:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: seq_num 2079 ranges {1843200=462848,2306048=745472,3051520=57344} 2014-07-01 17:05:27,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: writing mira0435601-223 from 1843200 to 2306048 tid 1 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: writing mira0435601-223 from 2306048 to 3051520 tid 2 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: writing mira0435601-223 from 3051520 to 3108864 tid 3 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 106 current snap is 0 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: seq_num 2080 ranges {3227648=86016} 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: writing mira0435601-106 from 3227648 to 3313664 tid 1 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3360: read oid 64 snap -1 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: finishing write tid 1 to mira0435601-337 2014-07-01 17:05:27,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: finishing write tid 3 to mira0435601-337 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: finishing write tid 4 to mira0435601-337 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 369 (ObjNum 2078 snap 0 seq_num 2078) dirty exists 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: finishing write tid 1 to mira0435601-223 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: finishing write tid 2 to mira0435601-223 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: finishing write tid 3 to mira0435601-223 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: finishing write tid 5 to mira0435601-223 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: done (3 left) 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3361: read oid 337 snap -1 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 282 current snap is 0 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: seq_num 2081 ranges {8482816=761856,9244672=802816,10047488=512000,10559488=290816} 2014-07-01 17:05:27,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: writing mira0435601-282 from 8482816 to 9244672 tid 1 2014-07-01 17:05:27,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: writing mira0435601-282 from 9244672 to 10047488 tid 2 2014-07-01 17:05:27,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: writing mira0435601-282 from 10047488 to 10559488 tid 3 2014-07-01 17:05:27,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: writing mira0435601-282 from 10559488 to 10850304 tid 4 2014-07-01 17:05:27,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: finishing write tid 6 to mira0435601-223 2014-07-01 17:05:27,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 222 (ObjNum 2079 snap 0 seq_num 2079) dirty exists 2014-07-01 17:05:27,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: finishing write tid 1 to mira0435601-106 2014-07-01 17:05:27,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: read oid 436 snap -1 2014-07-01 17:05:27,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: finishing write tid 3 to mira0435601-106 2014-07-01 17:05:27,921.921 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: finishing write tid 4 to mira0435601-106 2014-07-01 17:05:27,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 271 (ObjNum 2080 snap 0 seq_num 2080) dirty exists 2014-07-01 17:05:27,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3360: expect (ObjNum 2013 snap 0 seq_num 2013) 2014-07-01 17:05:28,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: expect (ObjNum 1182 snap 250 seq_num 1182) 2014-07-01 17:05:28,834.834 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.3. 2014-07-01 17:05:29,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 1 to mira0435601-282 2014-07-01 17:05:29,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 2 to mira0435601-282 2014-07-01 17:05:29,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 3 to mira0435601-282 2014-07-01 17:05:29,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 4 to mira0435601-282 2014-07-01 17:05:29,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 6 to mira0435601-282 2014-07-01 17:05:29,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3361: expect (ObjNum 2078 snap 0 seq_num 2078) 2014-07-01 17:05:29,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: finishing write tid 1 to mira0437250-129 2014-07-01 17:05:29,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: finishing write tid 2 to mira0437250-129 2014-07-01 17:05:29,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: finishing write tid 4 to mira0437250-129 2014-07-01 17:05:29,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: finishing write tid 5 to mira0437250-129 2014-07-01 17:05:29,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 1376 (ObjNum 1205 snap 254 seq_num 1205) dirty exists 2014-07-01 17:05:29,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2391: done (8 left) 2014-07-01 17:05:29,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2392: done (7 left) 2014-07-01 17:05:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2393: done (6 left) 2014-07-01 17:05:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2394: done (5 left) 2014-07-01 17:05:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2395: done (4 left) 2014-07-01 17:05:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2396: done (3 left) 2014-07-01 17:05:29,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2397: done (2 left) 2014-07-01 17:05:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2398: done (1 left) 2014-07-01 17:05:29,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2399: done (0 left) 2014-07-01 17:05:29,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: read oid 414 snap -1 2014-07-01 17:05:29,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: read oid 439 snap 222 2014-07-01 17:05:29,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2402: delete oid 335 current snap is 254 2014-07-01 17:05:29,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: expect (ObjNum 762 snap 112 seq_num 762) 2014-07-01 17:05:30,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: expect (ObjNum 438 snap 0 seq_num 438) 2014-07-01 17:05:31,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:2400: done (2 left) 2014-07-01 17:05:31,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:2401: done (1 left) 2014-07-01 17:05:31,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:2402: done (0 left) 2014-07-01 17:05:31,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: read oid 463 snap -1 2014-07-01 17:05:31,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2404: delete oid 479 current snap is 254 2014-07-01 17:05:31,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:2404: done (1 left) 2014-07-01 17:05:31,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: read oid 467 snap -1 2014-07-01 17:05:31,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: read oid 16 snap 242 2014-07-01 17:05:31,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:2407: rollback oid 324 current snap is 254 2014-07-01 17:05:31,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 324 to 251 2014-07-01 17:05:31,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: write oid 186 current snap is 254 2014-07-01 17:05:31,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: seq_num 1208 ranges {495318=672350,1598213=767672,2790225=484292,3295317=1} 2014-07-01 17:05:31,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: writing mira0437250-186 from 495318 to 1167668 tid 1 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: writing mira0437250-186 from 1598213 to 2365885 tid 2 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: writing mira0437250-186 from 2790225 to 3274517 tid 3 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: writing mira0437250-186 from 3295317 to 3295318 tid 4 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: read oid 479 snap 233 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: expect (ObjNum 1130 snap 230 seq_num 1130) 2014-07-01 17:05:31,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: expect (ObjNum 591 snap 39 seq_num 591) 2014-07-01 17:05:31,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: expect (ObjNum 616 snap 50 seq_num 616) 2014-07-01 17:05:31,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2403: done (5 left) 2014-07-01 17:05:31,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:2405: done (4 left) 2014-07-01 17:05:31,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2406: done (3 left) 2014-07-01 17:05:31,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:2407: done (2 left) 2014-07-01 17:05:31,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:2409: done (1 left) 2014-07-01 17:05:31,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: write oid 369 current snap is 254 2014-07-01 17:05:31,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: seq_num 1209 ranges {707924=644154,2077273=589291,3357402=150522} 2014-07-01 17:05:31,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: writing mira0437250-369 from 707924 to 1352078 tid 1 2014-07-01 17:05:31,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: writing mira0437250-369 from 2077273 to 2666564 tid 2 2014-07-01 17:05:32,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: writing mira0437250-369 from 3357402 to 3507924 tid 3 2014-07-01 17:05:32,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: write oid 448 current snap is 254 2014-07-01 17:05:32,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: seq_num 1210 ranges {403154=768406,1659181=659486,2911075=292079} 2014-07-01 17:05:32,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: writing mira0437250-448 from 403154 to 1171560 tid 1 2014-07-01 17:05:32,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: writing mira0437250-448 from 1659181 to 2318667 tid 2 2014-07-01 17:05:32,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: writing mira0437250-448 from 2911075 to 3203154 tid 3 2014-07-01 17:05:32,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: snap_create 2014-07-01 17:05:32,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 1 to mira0437250-186 2014-07-01 17:05:32,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 2 to mira0437250-186 2014-07-01 17:05:32,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 3 to mira0437250-186 2014-07-01 17:05:32,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 4 to mira0437250-186 2014-07-01 17:05:32,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 6 to mira0437250-186 2014-07-01 17:05:32,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: finishing write tid 1 to mira0437250-448 2014-07-01 17:05:32,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: finishing write tid 2 to mira0437250-448 2014-07-01 17:05:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: finishing write tid 3 to mira0437250-448 2014-07-01 17:05:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: finishing write tid 5 to mira0437250-448 2014-07-01 17:05:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: finishing write tid 6 to mira0437250-448 2014-07-01 17:05:32,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 448 v 1082 (ObjNum 1210 snap 254 seq_num 1210) dirty exists 2014-07-01 17:05:32,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2411: done (3 left) 2014-07-01 17:05:32,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2412: done (2 left) 2014-07-01 17:05:32,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: write oid 336 current snap is 255 2014-07-01 17:05:32,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: seq_num 1211 ranges {596559=400000} 2014-07-01 17:05:32,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: writing mira0437250-336 from 596559 to 996559 tid 1 2014-07-01 17:05:32,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: delete oid 242 current snap is 255 2014-07-01 17:05:33,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: finishing write tid 1 to mira0437250-336 2014-07-01 17:05:33,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: finishing write tid 3 to mira0437250-336 2014-07-01 17:05:33,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: finishing write tid 4 to mira0437250-336 2014-07-01 17:05:33,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 1084 (ObjNum 1211 snap 255 seq_num 1211) dirty exists 2014-07-01 17:05:33,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: finishing write tid 7 to mira0437250-186 2014-07-01 17:05:33,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 957 (ObjNum 1208 snap 254 seq_num 1208) dirty exists 2014-07-01 17:05:33,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: finishing write tid 1 to mira0437250-369 2014-07-01 17:05:33,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: finishing write tid 2 to mira0437250-369 2014-07-01 17:05:33,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: finishing write tid 3 to mira0437250-369 2014-07-01 17:05:33,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: finishing write tid 5 to mira0437250-369 2014-07-01 17:05:33,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: finishing write tid 6 to mira0437250-369 2014-07-01 17:05:33,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 369 v 754 (ObjNum 1209 snap 254 seq_num 1209) dirty exists 2014-07-01 17:05:33,845.845 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1] out_osds: [2, 3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:33,845.845 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:33,846.846 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 17:05:33,846.846 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:05:33,847.847 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:05:33,847.847 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 17:05:34,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:2408: done (3 left) 2014-07-01 17:05:34,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2410: done (2 left) 2014-07-01 17:05:34,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:2413: done (1 left) 2014-07-01 17:05:34,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2414: done (0 left) 2014-07-01 17:05:34,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2415: delete oid 243 current snap is 255 2014-07-01 17:05:34,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2415: done (0 left) 2014-07-01 17:05:34,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:2416: snap_create 2014-07-01 17:05:35,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:2416: done (0 left) 2014-07-01 17:05:35,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2417: rollback oid 151 current snap is 256 2014-07-01 17:05:35,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 151 to 197 2014-07-01 17:05:35,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: read oid 354 snap -1 2014-07-01 17:05:35,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: read oid 98 snap 197 2014-07-01 17:05:35,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: delete oid 2 current snap is 256 2014-07-01 17:05:35,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 1294 (ObjNum 150 snap 0 seq_num 150) dirty exists 2014-07-01 17:05:36,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: expect (ObjNum 97 snap 0 seq_num 97) 2014-07-01 17:05:38,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: finishing write tid 7 to mira0435601-282 2014-07-01 17:05:38,261.261 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 339 (ObjNum 2081 snap 0 seq_num 2081) dirty exists 2014-07-01 17:05:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: expect (ObjNum 1653 snap 0 seq_num 1653) 2014-07-01 17:05:38,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: expect (ObjNum 1181 snap 250 seq_num 1181) 2014-07-01 17:05:38,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:2417: done (3 left) 2014-07-01 17:05:38,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2418: done (2 left) 2014-07-01 17:05:38,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2419: done (1 left) 2014-07-01 17:05:38,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2420: done (0 left) 2014-07-01 17:05:38,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: write oid 237 current snap is 256 2014-07-01 17:05:38,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: seq_num 1215 ranges {793030=698596,2138450=254580} 2014-07-01 17:05:39,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: writing mira0437250-237 from 793030 to 1491626 tid 1 2014-07-01 17:05:39,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: writing mira0437250-237 from 2138450 to 2393030 tid 2 2014-07-01 17:05:39,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: snap_create 2014-07-01 17:05:39,273.273 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1] out_osds: [2, 3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:39,273.273 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:39,274.274 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:05:39,274.274 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:05:39,274.274 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:05:39,275.275 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:05:39,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: finishing write tid 1 to mira0437250-237 2014-07-01 17:05:39,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: finishing write tid 2 to mira0437250-237 2014-07-01 17:05:39,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: finishing write tid 4 to mira0437250-237 2014-07-01 17:05:39,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: finishing write tid 5 to mira0437250-237 2014-07-01 17:05:39,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 1088 (ObjNum 1215 snap 256 seq_num 1215) dirty exists 2014-07-01 17:05:40,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2421: done (1 left) 2014-07-01 17:05:40,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2422: done (0 left) 2014-07-01 17:05:40,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: write oid 5 current snap is 257 2014-07-01 17:05:40,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: seq_num 1216 ranges {488260=670400,1851558=236702} 2014-07-01 17:05:40,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: writing mira0437250-5 from 488260 to 1158660 tid 1 2014-07-01 17:05:40,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: writing mira0437250-5 from 1851558 to 2088260 tid 2 2014-07-01 17:05:40,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: read oid 459 snap -1 2014-07-01 17:05:40,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: snap_create 2014-07-01 17:05:40,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: expect (ObjNum 458 snap 0 seq_num 458) 2014-07-01 17:05:40,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 1 to mira0437250-5 2014-07-01 17:05:40,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 2 to mira0437250-5 2014-07-01 17:05:40,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 4 to mira0437250-5 2014-07-01 17:05:40,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: finishing write tid 5 to mira0437250-5 2014-07-01 17:05:40,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 726 (ObjNum 1216 snap 257 seq_num 1216) dirty exists 2014-07-01 17:05:41,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2423: done (2 left) 2014-07-01 17:05:41,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2424: done (1 left) 2014-07-01 17:05:41,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2425: done (0 left) 2014-07-01 17:05:41,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: read oid 107 snap -1 2014-07-01 17:05:41,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:2427: delete oid 18 current snap is 258 2014-07-01 17:05:41,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:2427: done (1 left) 2014-07-01 17:05:41,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:2428: snap_create 2014-07-01 17:05:41,084.084 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: expect (ObjNum 106 snap 0 seq_num 106) 2014-07-01 17:05:42,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: done (5 left) 2014-07-01 17:05:42,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: done (4 left) 2014-07-01 17:05:42,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:3360: done (3 left) 2014-07-01 17:05:42,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3361: done (2 left) 2014-07-01 17:05:42,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: done (1 left) 2014-07-01 17:05:42,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: done (0 left) 2014-07-01 17:05:42,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 95 current snap is 0 2014-07-01 17:05:42,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: seq_num 2082 ranges {1286144=630784,1916928=741376,2658304=737280,3395584=126976} 2014-07-01 17:05:42,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: writing mira0435601-95 from 1286144 to 1916928 tid 1 2014-07-01 17:05:42,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: writing mira0435601-95 from 1916928 to 2658304 tid 2 2014-07-01 17:05:42,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: writing mira0435601-95 from 2658304 to 3395584 tid 3 2014-07-01 17:05:42,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: writing mira0435601-95 from 3395584 to 3522560 tid 4 2014-07-01 17:05:42,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 157 current snap is 0 2014-07-01 17:05:42,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: seq_num 2083 ranges {3682304=708608,4390912=569344,4960256=614400,5574656=20480} 2014-07-01 17:05:42,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0435601-157 from 3682304 to 4390912 tid 1 2014-07-01 17:05:42,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0435601-157 from 4390912 to 4960256 tid 2 2014-07-01 17:05:42,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0435601-157 from 4960256 to 5574656 tid 3 2014-07-01 17:05:42,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0435601-157 from 5574656 to 5595136 tid 4 2014-07-01 17:05:42,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 111 current snap is 0 2014-07-01 17:05:42,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: seq_num 2084 ranges {2560000=581632,3141632=552960,3694592=651264} 2014-07-01 17:05:42,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0435601-111 from 2560000 to 3141632 tid 1 2014-07-01 17:05:42,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0435601-111 from 3141632 to 3694592 tid 2 2014-07-01 17:05:42,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0435601-111 from 3694592 to 4345856 tid 3 2014-07-01 17:05:42,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: read oid 281 snap -1 2014-07-01 17:05:42,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 1 to mira0435601-95 2014-07-01 17:05:42,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 2 to mira0435601-95 2014-07-01 17:05:42,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 3 to mira0435601-95 2014-07-01 17:05:42,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 4 to mira0435601-95 2014-07-01 17:05:42,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 6 to mira0435601-95 2014-07-01 17:05:42,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 164 current snap is 0 2014-07-01 17:05:42,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: seq_num 2085 ranges {2719744=606208} 2014-07-01 17:05:42,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: writing mira0435601-164 from 2719744 to 3325952 tid 1 2014-07-01 17:05:42,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:3369: read oid 119 snap -1 2014-07-01 17:05:42,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:3370: delete oid 45 current snap is 0 2014-07-01 17:05:42,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: expect (ObjNum 1855 snap 0 seq_num 1855) 2014-07-01 17:05:42,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2426: done (1 left) 2014-07-01 17:05:43,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2428: done (0 left) 2014-07-01 17:05:43,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: read oid 30 snap -1 2014-07-01 17:05:43,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: snap_create 2014-07-01 17:05:43,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: expect (ObjNum 29 snap 0 seq_num 29) 2014-07-01 17:05:43,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2429: done (1 left) 2014-07-01 17:05:43,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2430: done (0 left) 2014-07-01 17:05:43,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: read oid 113 snap -1 2014-07-01 17:05:43,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: rollback oid 21 current snap is 260 2014-07-01 17:05:43,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 21 to 247 2014-07-01 17:05:43,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: snap_remove snap 197 2014-07-01 17:05:43,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 21 v 0 (ObjNum 2 snap 0 seq_num 1) dirty dne 2014-07-01 17:05:43,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: expect (ObjNum 719 snap 97 seq_num 719) 2014-07-01 17:05:44,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 1 to mira0435601-111 2014-07-01 17:05:44,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 2 to mira0435601-111 2014-07-01 17:05:44,111.111 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 3 to mira0435601-111 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: finishing write tid 1 to mira0435601-164 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: finishing write tid 3 to mira0435601-164 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 5 to mira0435601-111 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: finishing write tid 7 to mira0435601-95 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 328 (ObjNum 2082 snap 0 seq_num 2082) dirty exists 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: finishing write tid 4 to mira0435601-164 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 164 v 330 (ObjNum 2085 snap 0 seq_num 2085) dirty exists 2014-07-01 17:05:44,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3369: expect (ObjNum 1329 snap 0 seq_num 1329) 2014-07-01 17:05:44,744.744 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1] out_osds: [2, 3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:44,744.744 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:44,744.744 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2014-07-01 17:05:44,745.745 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 1' 2014-07-01 17:05:45,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:2431: done (2 left) 2014-07-01 17:05:45,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:2432: done (1 left) 2014-07-01 17:05:45,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2433: done (0 left) 2014-07-01 17:05:45,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: read oid 386 snap -1 2014-07-01 17:05:45,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: read oid 33 snap 244 2014-07-01 17:05:45,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:2436: read oid 179 snap -1 2014-07-01 17:05:45,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2437: rollback oid 6 current snap is 260 2014-07-01 17:05:45,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 6 to 238 2014-07-01 17:05:45,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: read oid 383 snap -1 2014-07-01 17:05:45,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: delete oid 44 current snap is 260 2014-07-01 17:05:45,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2439: done (5 left) 2014-07-01 17:05:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: write oid 434 current snap is 260 2014-07-01 17:05:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: seq_num 1219 ranges {589501=621910,2007459=428139,3199032=590469} 2014-07-01 17:05:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0437250-434 from 589501 to 1211411 tid 1 2014-07-01 17:05:45,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0437250-434 from 2007459 to 2435598 tid 2 2014-07-01 17:05:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: writing mira0437250-434 from 3199032 to 3789501 tid 3 2014-07-01 17:05:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2441: snap_remove snap 233 2014-07-01 17:05:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: expect (ObjNum 947 snap 173 seq_num 947) 2014-07-01 17:05:45,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 0 (ObjNum 16784704 snap 0 seq_num 40031872) dirty dne 2014-07-01 17:05:45,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: expect (ObjNum 1094 snap 220 seq_num 1094) 2014-07-01 17:05:46,323.323 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 1 (8655362) 2014-07-01 17:05:46,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 6 to mira0435601-111 2014-07-01 17:05:46,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 215 (ObjNum 2084 snap 0 seq_num 2084) dirty exists 2014-07-01 17:05:46,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 1 to mira0435601-157 2014-07-01 17:05:46,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 2 to mira0435601-157 2014-07-01 17:05:46,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 3 to mira0435601-157 2014-07-01 17:05:46,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 4 to mira0435601-157 2014-07-01 17:05:46,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 6 to mira0435601-157 2014-07-01 17:05:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 7 to mira0435601-157 2014-07-01 17:05:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 300 (ObjNum 2083 snap 0 seq_num 2083) dirty exists 2014-07-01 17:05:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: done (6 left) 2014-07-01 17:05:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: done (5 left) 2014-07-01 17:05:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: done (4 left) 2014-07-01 17:05:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: done (3 left) 2014-07-01 17:05:46,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: done (2 left) 2014-07-01 17:05:46,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3369: done (1 left) 2014-07-01 17:05:46,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3370: done (0 left) 2014-07-01 17:05:47,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:3371: read oid 59 snap -1 2014-07-01 17:05:47,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3372: read oid 366 snap -1 2014-07-01 17:05:47,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: read oid 266 snap -1 2014-07-01 17:05:47,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 17:05:47,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: seq_num 2087 ranges {6782976=352256} 2014-07-01 17:05:47,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: writing mira0435601-400 from 6782976 to 7135232 tid 1 2014-07-01 17:05:47,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 390 current snap is 0 2014-07-01 17:05:47,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: seq_num 2088 ranges {53248=24576} 2014-07-01 17:05:47,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: writing mira0435601-390 from 53248 to 77824 tid 1 2014-07-01 17:05:47,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: read oid 212 snap -1 2014-07-01 17:05:47,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3377: delete oid 337 current snap is 0 2014-07-01 17:05:47,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3372: expect (ObjNum 1285 snap 0 seq_num 1285) 2014-07-01 17:05:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 1 to mira0437250-434 2014-07-01 17:05:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 2 to mira0437250-434 2014-07-01 17:05:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 3 to mira0437250-434 2014-07-01 17:05:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 5 to mira0437250-434 2014-07-01 17:05:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: finishing write tid 6 to mira0437250-434 2014-07-01 17:05:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 818 (ObjNum 1219 snap 260 seq_num 1219) dirty exists 2014-07-01 17:05:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2434: done (6 left) 2014-07-01 17:05:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2435: done (5 left) 2014-07-01 17:05:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:2436: done (4 left) 2014-07-01 17:05:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2437: done (3 left) 2014-07-01 17:05:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2438: done (2 left) 2014-07-01 17:05:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2440: done (1 left) 2014-07-01 17:05:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2441: done (0 left) 2014-07-01 17:05:47,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: write oid 374 current snap is 260 2014-07-01 17:05:47,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: seq_num 1220 ranges {533306=1} 2014-07-01 17:05:47,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: writing mira0437250-374 from 533306 to 533307 tid 1 2014-07-01 17:05:47,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: read oid 373 snap -1 2014-07-01 17:05:47,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: read oid 434 snap -1 2014-07-01 17:05:47,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2445: snap_create 2014-07-01 17:05:47,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: expect (ObjNum 372 snap 0 seq_num 372) 2014-07-01 17:05:47,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: expect (ObjNum 1219 snap 260 seq_num 1219) 2014-07-01 17:05:48,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: expect (ObjNum 1908 snap 0 seq_num 1908) 2014-07-01 17:05:49,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 1 to mira0437250-374 2014-07-01 17:05:49,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 3 to mira0437250-374 2014-07-01 17:05:49,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: finishing write tid 4 to mira0437250-374 2014-07-01 17:05:49,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 374 v 1402 (ObjNum 1220 snap 260 seq_num 1220) dirty exists 2014-07-01 17:05:49,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2442: done (3 left) 2014-07-01 17:05:49,685.685 INFO:teuthology.task.rados.rados.0.mira043.stdout:2443: done (2 left) 2014-07-01 17:05:49,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2444: done (1 left) 2014-07-01 17:05:49,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2445: done (0 left) 2014-07-01 17:05:49,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: write oid 347 current snap is 261 2014-07-01 17:05:49,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: seq_num 1221 ranges {478137=717966,1278136=1} 2014-07-01 17:05:49,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: writing mira0437250-347 from 478137 to 1196103 tid 1 2014-07-01 17:05:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: writing mira0437250-347 from 1278136 to 1278137 tid 2 2014-07-01 17:05:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2447: snap_create 2014-07-01 17:05:49,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: finishing write tid 1 to mira0437250-347 2014-07-01 17:05:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: finishing write tid 2 to mira0437250-347 2014-07-01 17:05:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: finishing write tid 4 to mira0437250-347 2014-07-01 17:05:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: finishing write tid 5 to mira0437250-347 2014-07-01 17:05:49,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 347 v 970 (ObjNum 1221 snap 261 seq_num 1221) dirty exists 2014-07-01 17:05:50,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2446: done (1 left) 2014-07-01 17:05:50,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2447: done (0 left) 2014-07-01 17:05:50,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: write oid 101 current snap is 262 2014-07-01 17:05:50,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: seq_num 1222 ranges {690742=442219,1490741=1} 2014-07-01 17:05:50,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: writing mira0437250-101 from 690742 to 1132961 tid 1 2014-07-01 17:05:50,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: writing mira0437250-101 from 1490741 to 1490742 tid 2 2014-07-01 17:05:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2449: snap_create 2014-07-01 17:05:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 1 to mira0437250-101 2014-07-01 17:05:50,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 2 to mira0437250-101 2014-07-01 17:05:50,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 4 to mira0437250-101 2014-07-01 17:05:50,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: finishing write tid 5 to mira0437250-101 2014-07-01 17:05:50,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 978 (ObjNum 1222 snap 262 seq_num 1222) dirty exists 2014-07-01 17:05:50,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: expect (ObjNum 976 snap 0 seq_num 976) 2014-07-01 17:05:51,332.332 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1] out_osds: [2, 3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:51,332.332 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:51,332.332 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 17:05:51,333.333 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:05:51,333.333 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:05:51,333.333 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 17:05:51,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2448: done (1 left) 2014-07-01 17:05:51,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2449: done (0 left) 2014-07-01 17:05:51,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: read oid 125 snap -1 2014-07-01 17:05:51,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2451: rollback oid 32 current snap is 263 2014-07-01 17:05:51,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 32 to 156 2014-07-01 17:05:51,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: write oid 59 current snap is 263 2014-07-01 17:05:51,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: seq_num 1223 ranges {634548=697671,1992819=241729} 2014-07-01 17:05:51,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: writing mira0437250-59 from 634548 to 1332219 tid 1 2014-07-01 17:05:51,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: writing mira0437250-59 from 1992819 to 2234548 tid 2 2014-07-01 17:05:51,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2453: rollback oid 322 current snap is 263 2014-07-01 17:05:51,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 322 to 261 2014-07-01 17:05:51,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 32 v 0 (ObjNum 2 snap 0 seq_num 0) dirty dne 2014-07-01 17:05:51,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2451: done (3 left) 2014-07-01 17:05:51,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: snap_remove snap 239 2014-07-01 17:05:51,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 0 (ObjNum -1855068216 snap 32691 seq_num 17424208) dirty dne 2014-07-01 17:05:51,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: expect (ObjNum 506 snap 3 seq_num 506) 2014-07-01 17:05:51,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3371: expect (ObjNum 1981 snap 0 seq_num 1981) 2014-07-01 17:05:52,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 1 to mira0435601-400 2014-07-01 17:05:52,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 3 to mira0435601-400 2014-07-01 17:05:52,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: finishing write tid 1 to mira0435601-390 2014-07-01 17:05:52,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: finishing write tid 3 to mira0435601-390 2014-07-01 17:05:52,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: finishing write tid 4 to mira0435601-390 2014-07-01 17:05:52,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 390 v 174 (ObjNum 2088 snap 0 seq_num 2088) dirty exists 2014-07-01 17:05:52,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 4 to mira0435601-400 2014-07-01 17:05:52,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 323 (ObjNum 2087 snap 0 seq_num 2087) dirty exists 2014-07-01 17:05:52,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:3371: done (6 left) 2014-07-01 17:05:52,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:3372: done (5 left) 2014-07-01 17:05:52,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: done (4 left) 2014-07-01 17:05:52,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: done (3 left) 2014-07-01 17:05:52,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: done (2 left) 2014-07-01 17:05:52,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: done (1 left) 2014-07-01 17:05:52,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:3377: done (0 left) 2014-07-01 17:05:52,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 392 current snap is 0 2014-07-01 17:05:53,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: seq_num 2090 ranges {2187264=573440,2760704=700416,3461120=679936,4141056=225280} 2014-07-01 17:05:53,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: writing mira0435601-392 from 2187264 to 2760704 tid 1 2014-07-01 17:05:53,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: writing mira0435601-392 from 2760704 to 3461120 tid 2 2014-07-01 17:05:53,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: writing mira0435601-392 from 3461120 to 4141056 tid 3 2014-07-01 17:05:53,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: writing mira0435601-392 from 4141056 to 4366336 tid 4 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3379: read oid 281 snap -1 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 1 to mira0435601-392 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 2 to mira0435601-392 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 373 current snap is 0 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: seq_num 2091 ranges {0=598016,598016=401408} 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: writing mira0435601-373 from 0 to 598016 tid 1 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: writing mira0435601-373 from 598016 to 999424 tid 2 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 38 current snap is 0 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: seq_num 2092 ranges {2895872=520192,3416064=761856,4177920=442368} 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: writing mira0435601-38 from 2895872 to 3416064 tid 1 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: writing mira0435601-38 from 3416064 to 4177920 tid 2 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: writing mira0435601-38 from 4177920 to 4620288 tid 3 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 17:05:53,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: seq_num 2093 ranges {2318336=794624,3112960=745472,3858432=57344} 2014-07-01 17:05:53,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: writing mira0435601-273 from 2318336 to 3112960 tid 1 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: writing mira0435601-273 from 3112960 to 3858432 tid 2 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: writing mira0435601-273 from 3858432 to 3915776 tid 3 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3383: read oid 138 snap -1 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 3 to mira0435601-392 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 4 to mira0435601-392 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 6 to mira0435601-392 2014-07-01 17:05:53,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3379: expect (ObjNum 1855 snap 0 seq_num 1855) 2014-07-01 17:05:53,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 1 to mira0437250-59 2014-07-01 17:05:53,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 2 to mira0437250-59 2014-07-01 17:05:53,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 4 to mira0437250-59 2014-07-01 17:05:53,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: finishing write tid 5 to mira0437250-59 2014-07-01 17:05:53,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 891 (ObjNum 1223 snap 263 seq_num 1223) dirty exists 2014-07-01 17:05:53,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:2450: done (3 left) 2014-07-01 17:05:53,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:2452: done (2 left) 2014-07-01 17:05:53,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:2453: done (1 left) 2014-07-01 17:05:53,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:2454: done (0 left) 2014-07-01 17:05:53,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: snap_remove snap 261 2014-07-01 17:05:53,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2455: done (0 left) 2014-07-01 17:05:53,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: read oid 272 snap -1 2014-07-01 17:05:53,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: write oid 35 current snap is 263 2014-07-01 17:05:53,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: seq_num 1224 ranges {578354=420899,1554503=681963,2701287=277067} 2014-07-01 17:05:53,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0437250-35 from 578354 to 999253 tid 1 2014-07-01 17:05:53,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0437250-35 from 1554503 to 2236466 tid 2 2014-07-01 17:05:53,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: writing mira0437250-35 from 2701287 to 2978354 tid 3 2014-07-01 17:05:53,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: snap_remove snap 242 2014-07-01 17:05:53,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 1 to mira0437250-35 2014-07-01 17:05:53,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 2 to mira0437250-35 2014-07-01 17:05:53,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 3 to mira0437250-35 2014-07-01 17:05:53,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 5 to mira0437250-35 2014-07-01 17:05:53,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: finishing write tid 6 to mira0437250-35 2014-07-01 17:05:53,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 1161 (ObjNum 1224 snap 263 seq_num 1224) dirty exists 2014-07-01 17:05:54,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 7 to mira0435601-392 2014-07-01 17:05:54,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 392 v 279 (ObjNum 2090 snap 0 seq_num 2090) dirty exists 2014-07-01 17:05:54,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: finishing write tid 1 to mira0435601-373 2014-07-01 17:05:54,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: finishing write tid 2 to mira0435601-373 2014-07-01 17:05:54,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: finishing write tid 4 to mira0435601-373 2014-07-01 17:05:54,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: finishing write tid 5 to mira0435601-373 2014-07-01 17:05:54,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 326 (ObjNum 2091 snap 0 seq_num 2091) dirty exists 2014-07-01 17:05:54,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: finishing write tid 1 to mira0435601-38 2014-07-01 17:05:54,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: done (5 left) 2014-07-01 17:05:54,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3379: done (4 left) 2014-07-01 17:05:54,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: done (3 left) 2014-07-01 17:05:54,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3384: read oid 233 snap -1 2014-07-01 17:05:54,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: finishing write tid 2 to mira0435601-38 2014-07-01 17:05:54,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: finishing write tid 3 to mira0435601-38 2014-07-01 17:05:54,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: finishing write tid 5 to mira0435601-38 2014-07-01 17:05:54,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: finishing write tid 6 to mira0435601-38 2014-07-01 17:05:54,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 38 v 334 (ObjNum 2092 snap 0 seq_num 2092) dirty exists 2014-07-01 17:05:54,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: finishing write tid 1 to mira0435601-273 2014-07-01 17:05:54,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: finishing write tid 2 to mira0435601-273 2014-07-01 17:05:54,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: finishing write tid 3 to mira0435601-273 2014-07-01 17:05:54,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: finishing write tid 5 to mira0435601-273 2014-07-01 17:05:54,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: finishing write tid 6 to mira0435601-273 2014-07-01 17:05:54,338.338 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 191 (ObjNum 2093 snap 0 seq_num 2093) dirty exists 2014-07-01 17:05:54,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: done (3 left) 2014-07-01 17:05:54,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: done (2 left) 2014-07-01 17:05:54,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3383: done (1 left) 2014-07-01 17:05:54,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 488 current snap is 0 2014-07-01 17:05:54,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: seq_num 2094 ranges {1867776=417792} 2014-07-01 17:05:54,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: writing mira0435601-488 from 1867776 to 2285568 tid 1 2014-07-01 17:05:54,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3386: read oid 365 snap -1 2014-07-01 17:05:54,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: delete oid 118 current snap is 0 2014-07-01 17:05:54,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3384: expect (ObjNum 1811 snap 0 seq_num 1811) 2014-07-01 17:05:54,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2456: done (2 left) 2014-07-01 17:05:54,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2457: done (1 left) 2014-07-01 17:05:54,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2458: done (0 left) 2014-07-01 17:05:54,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:2459: snap_remove snap 249 2014-07-01 17:05:55,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:2459: done (0 left) 2014-07-01 17:05:55,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: read oid 271 snap -1 2014-07-01 17:05:55,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: snap_create 2014-07-01 17:05:55,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: expect (ObjNum 270 snap 0 seq_num 270) 2014-07-01 17:05:55,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: finishing write tid 1 to mira0435601-488 2014-07-01 17:05:55,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: finishing write tid 3 to mira0435601-488 2014-07-01 17:05:55,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3386: expect (ObjNum 2061 snap 0 seq_num 2061) 2014-07-01 17:05:56,929.929 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1] out_osds: [2, 3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:05:56,930.930 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:05:56,931.931 INFO:teuthology.task.thrashosds.thrasher:Adding osd 2 2014-07-01 17:05:56,932.932 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 2' 2014-07-01 17:05:57,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: finishing write tid 4 to mira0435601-488 2014-07-01 17:05:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 165 (ObjNum 2094 snap 0 seq_num 2094) dirty exists 2014-07-01 17:05:57,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3384: done (3 left) 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: done (2 left) 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3386: done (1 left) 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: done (0 left) 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3388: read oid 430 snap -1 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3389: read oid 68 snap -1 2014-07-01 17:05:57,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 154 current snap is 0 2014-07-01 17:05:57,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: seq_num 2096 ranges {4218880=163840} 2014-07-01 17:05:57,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: writing mira0435601-154 from 4218880 to 4382720 tid 1 2014-07-01 17:05:57,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: read oid 107 snap -1 2014-07-01 17:05:57,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3392: read oid 76 snap -1 2014-07-01 17:05:57,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3393: read oid 50 snap -1 2014-07-01 17:05:57,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3394: delete oid 414 current snap is 0 2014-07-01 17:05:57,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3389: expect (ObjNum 1706 snap 0 seq_num 1706) 2014-07-01 17:05:57,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2460: done (1 left) 2014-07-01 17:05:57,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2461: done (0 left) 2014-07-01 17:05:57,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: write oid 44 current snap is 264 2014-07-01 17:05:57,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: seq_num 1225 ranges {522160=793728,2033564=635806,3400689=321471} 2014-07-01 17:05:57,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0437250-44 from 522160 to 1315888 tid 1 2014-07-01 17:05:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0437250-44 from 2033564 to 2669370 tid 2 2014-07-01 17:05:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: writing mira0437250-44 from 3400689 to 3722160 tid 3 2014-07-01 17:05:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: write oid 325 current snap is 264 2014-07-01 17:05:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: seq_num 1226 ranges {618413=516956,1899119=706001,3238009=180404} 2014-07-01 17:05:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0437250-325 from 618413 to 1135369 tid 1 2014-07-01 17:05:57,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0437250-325 from 1899119 to 2605120 tid 2 2014-07-01 17:05:57,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: writing mira0437250-325 from 3238009 to 3418413 tid 3 2014-07-01 17:05:57,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: write oid 145 current snap is 264 2014-07-01 17:05:57,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: seq_num 1227 ranges {679594=1} 2014-07-01 17:05:57,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: writing mira0437250-145 from 679594 to 679595 tid 1 2014-07-01 17:05:57,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: write oid 116 current snap is 264 2014-07-01 17:05:57,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: seq_num 1228 ranges {623401=745237,1423400=1} 2014-07-01 17:05:57,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: writing mira0437250-116 from 623401 to 1368638 tid 1 2014-07-01 17:05:57,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: writing mira0437250-116 from 1423400 to 1423401 tid 2 2014-07-01 17:05:57,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2466: delete oid 121 current snap is 264 2014-07-01 17:05:57,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 1 to mira0437250-44 2014-07-01 17:05:57,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 2 to mira0437250-44 2014-07-01 17:05:57,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 3 to mira0437250-44 2014-07-01 17:05:57,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 5 to mira0437250-44 2014-07-01 17:05:57,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: finishing write tid 6 to mira0437250-44 2014-07-01 17:05:57,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 44 v 940 (ObjNum 1225 snap 264 seq_num 1225) dirty exists 2014-07-01 17:05:57,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 1 to mira0437250-325 2014-07-01 17:05:57,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 2 to mira0437250-325 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 3 to mira0437250-325 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 5 to mira0437250-325 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: finishing write tid 1 to mira0437250-145 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: finishing write tid 3 to mira0437250-145 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: finishing write tid 6 to mira0437250-325 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 1003 (ObjNum 1226 snap 264 seq_num 1226) dirty exists 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: finishing write tid 4 to mira0437250-145 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 743 (ObjNum 1227 snap 264 seq_num 1227) dirty exists 2014-07-01 17:05:57,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2462: done (4 left) 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2463: done (3 left) 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2464: done (2 left) 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2466: done (1 left) 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: snap_remove snap 251 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 1 to mira0437250-116 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 2 to mira0437250-116 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 4 to mira0437250-116 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: finishing write tid 5 to mira0437250-116 2014-07-01 17:05:57,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 1412 (ObjNum 1228 snap 264 seq_num 1228) dirty exists 2014-07-01 17:05:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:2465: done (1 left) 2014-07-01 17:05:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:2467: done (0 left) 2014-07-01 17:05:58,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:2468: snap_create 2014-07-01 17:05:58,479.479 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.2. 2014-07-01 17:05:58,498.498 INFO:teuthology.task.thrashosds.thrasher:Added osd 2 2014-07-01 17:05:59,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2468: done (0 left) 2014-07-01 17:05:59,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2469: snap_remove snap 185 2014-07-01 17:05:59,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3388: expect (ObjNum 1587 snap 0 seq_num 1587) 2014-07-01 17:06:00,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:2469: done (0 left) 2014-07-01 17:06:00,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2470: read oid 454 snap -1 2014-07-01 17:06:00,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: read oid 335 snap -1 2014-07-01 17:06:00,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: write oid 125 current snap is 265 2014-07-01 17:06:00,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: seq_num 1230 ranges {414884=400000} 2014-07-01 17:06:00,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: writing mira0437250-125 from 414884 to 814884 tid 1 2014-07-01 17:06:00,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2473: snap_create 2014-07-01 17:06:00,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2470: expect (ObjNum 1201 snap 254 seq_num 1201) 2014-07-01 17:06:02,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:2470: done (3 left) 2014-07-01 17:06:02,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2471: done (2 left) 2014-07-01 17:06:02,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2473: done (1 left) 2014-07-01 17:06:02,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: snap_create 2014-07-01 17:06:02,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2474: done (1 left) 2014-07-01 17:06:02,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2475: snap_create 2014-07-01 17:06:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: finishing write tid 1 to mira0435601-154 2014-07-01 17:06:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: finishing write tid 3 to mira0435601-154 2014-07-01 17:06:02,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: finishing write tid 4 to mira0435601-154 2014-07-01 17:06:02,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 373 (ObjNum 2096 snap 0 seq_num 2096) dirty exists 2014-07-01 17:06:02,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:3392: expect (ObjNum 1962 snap 0 seq_num 1962) 2014-07-01 17:06:03,497.497 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 2] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:03,498.498 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:03,498.498 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 17:06:03,499.499 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:06:03,499.499 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:06:03,499.499 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 17:06:03,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2475: done (1 left) 2014-07-01 17:06:03,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2476: snap_remove snap 241 2014-07-01 17:06:04,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2476: done (1 left) 2014-07-01 17:06:04,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: snap_create 2014-07-01 17:06:05,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: expect (ObjNum 1632 snap 0 seq_num 1632) 2014-07-01 17:06:05,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2477: done (1 left) 2014-07-01 17:06:05,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: snap_remove snap 247 2014-07-01 17:06:06,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:2478: done (1 left) 2014-07-01 17:06:06,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: rollback oid 373 current snap is 269 2014-07-01 17:06:06,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 373 to 156 2014-07-01 17:06:06,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: write oid 480 current snap is 269 2014-07-01 17:06:06,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: seq_num 1231 ranges {724642=695722,1943317=559825,2953769=170873} 2014-07-01 17:06:06,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: writing mira0437250-480 from 724642 to 1420364 tid 1 2014-07-01 17:06:06,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: writing mira0437250-480 from 1943317 to 2503142 tid 2 2014-07-01 17:06:06,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: writing mira0437250-480 from 2953769 to 3124642 tid 3 2014-07-01 17:06:06,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: write oid 289 current snap is 269 2014-07-01 17:06:06,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: seq_num 1232 ranges {419872=419974,1408873=628996,2789042=30830} 2014-07-01 17:06:06,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: writing mira0437250-289 from 419872 to 839846 tid 1 2014-07-01 17:06:06,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: writing mira0437250-289 from 1408873 to 2037869 tid 2 2014-07-01 17:06:06,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: writing mira0437250-289 from 2789042 to 2819872 tid 3 2014-07-01 17:06:06,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2482: snap_remove snap 253 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3388: done (6 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3389: done (5 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: done (4 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: done (3 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3392: done (2 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3393: done (1 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3394: done (0 left) 2014-07-01 17:06:06,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 343 current snap is 0 2014-07-01 17:06:06,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: seq_num 2098 ranges {868352=667648,1536000=401408} 2014-07-01 17:06:06,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: writing mira0435601-343 from 868352 to 1536000 tid 1 2014-07-01 17:06:06,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: writing mira0435601-343 from 1536000 to 1937408 tid 2 2014-07-01 17:06:06,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3396: read oid 274 snap -1 2014-07-01 17:06:06,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3397: read oid 381 snap -1 2014-07-01 17:06:06,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: read oid 183 snap -1 2014-07-01 17:06:06,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 452 current snap is 0 2014-07-01 17:06:06,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: seq_num 2099 ranges {147456=786432,933888=491520,1425408=712704} 2014-07-01 17:06:06,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: writing mira0435601-452 from 147456 to 933888 tid 1 2014-07-01 17:06:06,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: writing mira0435601-452 from 933888 to 1425408 tid 2 2014-07-01 17:06:06,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: writing mira0435601-452 from 1425408 to 2138112 tid 3 2014-07-01 17:06:06,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: read oid 469 snap -1 2014-07-01 17:06:06,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3396: expect (ObjNum 1831 snap 0 seq_num 1831) 2014-07-01 17:06:07,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2482: done (4 left) 2014-07-01 17:06:07,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:2483: delete oid 12 current snap is 269 2014-07-01 17:06:07,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: expect (ObjNum 182 snap 0 seq_num 182) 2014-07-01 17:06:08,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3397: expect (ObjNum 1857 snap 0 seq_num 1857) 2014-07-01 17:06:08,937.937 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 2] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:08,937.937 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:08,938.938 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:06:08,938.938 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:06:08,938.938 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:06:08,939.939 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:06:10,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 1 to mira0435601-343 2014-07-01 17:06:10,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 2 to mira0435601-343 2014-07-01 17:06:10,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 4 to mira0435601-343 2014-07-01 17:06:10,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 5 to mira0435601-343 2014-07-01 17:06:10,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 337 (ObjNum 2098 snap 0 seq_num 2098) dirty exists 2014-07-01 17:06:10,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: done (5 left) 2014-07-01 17:06:10,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3396: done (4 left) 2014-07-01 17:06:10,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3397: done (3 left) 2014-07-01 17:06:10,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: done (2 left) 2014-07-01 17:06:10,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 114 current snap is 0 2014-07-01 17:06:10,453.453 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: seq_num 2100 ranges {2822144=409600,3231744=401408} 2014-07-01 17:06:10,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: writing mira0435601-114 from 2822144 to 3231744 tid 1 2014-07-01 17:06:10,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: writing mira0435601-114 from 3231744 to 3633152 tid 2 2014-07-01 17:06:10,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3402: read oid 315 snap -1 2014-07-01 17:06:10,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: read oid 301 snap -1 2014-07-01 17:06:10,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3404: read oid 202 snap -1 2014-07-01 17:06:10,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: read oid 465 snap -1 2014-07-01 17:06:10,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3406: read oid 210 snap -1 2014-07-01 17:06:10,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 134 current snap is 0 2014-07-01 17:06:10,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: seq_num 2101 ranges {0=487424} 2014-07-01 17:06:10,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: writing mira0435601-134 from 0 to 487424 tid 1 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: read oid 380 snap -1 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3402: done (9 left) 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: done (8 left) 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3406: done (7 left) 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: seq_num 2102 ranges {0=602112,602112=733184,1335296=69632} 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: writing mira0435601-382 from 0 to 602112 tid 1 2014-07-01 17:06:10,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: writing mira0435601-382 from 602112 to 1335296 tid 2 2014-07-01 17:06:10,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: writing mira0435601-382 from 1335296 to 1404928 tid 3 2014-07-01 17:06:10,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: read oid 319 snap -1 2014-07-01 17:06:10,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3404: expect (ObjNum 1836 snap 0 seq_num 1836) 2014-07-01 17:06:10,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: expect (ObjNum 2052 snap 0 seq_num 2052) 2014-07-01 17:06:12,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 17:06:13,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: expect (ObjNum 1879 snap 0 seq_num 1879) 2014-07-01 17:06:14,347.347 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 1, 2] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:14,347.347 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:14,347.347 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [5, 4, 0, 1, 2] 2014-07-01 17:06:14,347.347 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 17:06:14,930.930 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 17:06:16,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 1 to mira0437250-125 2014-07-01 17:06:16,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 3 to mira0437250-125 2014-07-01 17:06:16,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2483: done (4 left) 2014-07-01 17:06:16,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: write oid 305 current snap is 269 2014-07-01 17:06:16,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: seq_num 1234 ranges {308506=1} 2014-07-01 17:06:16,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: writing mira0437250-305 from 308506 to 308507 tid 1 2014-07-01 17:06:16,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: write oid 119 current snap is 269 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: seq_num 1235 ranges {3736=1} 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: writing mira0437250-119 from 3736 to 3737 tid 1 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: read oid 368 snap 246 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2487: read oid 108 snap 250 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: snap_create 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: finishing write tid 4 to mira0437250-125 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 587 (ObjNum 1230 snap 265 seq_num 1230) dirty exists 2014-07-01 17:06:16,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: expect (ObjNum 367 snap 0 seq_num 367) 2014-07-01 17:06:17,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: finishing write tid 1 to mira0435601-134 2014-07-01 17:06:17,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: finishing write tid 3 to mira0435601-134 2014-07-01 17:06:17,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: finishing write tid 4 to mira0435601-134 2014-07-01 17:06:17,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 378 (ObjNum 2101 snap 0 seq_num 2101) dirty exists 2014-07-01 17:06:17,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: done (8 left) 2014-07-01 17:06:17,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3404: done (7 left) 2014-07-01 17:06:17,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: done (6 left) 2014-07-01 17:06:17,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: done (5 left) 2014-07-01 17:06:17,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: done (4 left) 2014-07-01 17:06:17,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 27 current snap is 0 2014-07-01 17:06:18,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: seq_num 2103 ranges {11800576=229376} 2014-07-01 17:06:18,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: writing mira0435601-27 from 11800576 to 12029952 tid 1 2014-07-01 17:06:18,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3412: read oid 339 snap -1 2014-07-01 17:06:18,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 461 current snap is 0 2014-07-01 17:06:18,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: seq_num 2104 ranges {2179072=659456,2838528=548864,3387392=253952} 2014-07-01 17:06:18,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2487: expect (ObjNum 107 snap 0 seq_num 107) 2014-07-01 17:06:18,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: writing mira0435601-461 from 2179072 to 2838528 tid 1 2014-07-01 17:06:18,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: writing mira0435601-461 from 2838528 to 3387392 tid 2 2014-07-01 17:06:18,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: writing mira0435601-461 from 3387392 to 3641344 tid 3 2014-07-01 17:06:18,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3414: read oid 418 snap -1 2014-07-01 17:06:18,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3412: expect (ObjNum 2000 snap 0 seq_num 2000) 2014-07-01 17:06:18,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3414: expect (ObjNum 1929 snap 0 seq_num 1929) 2014-07-01 17:06:19,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 1 to mira0437250-305 2014-07-01 17:06:19,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 3 to mira0437250-305 2014-07-01 17:06:19,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: finishing write tid 4 to mira0437250-305 2014-07-01 17:06:19,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 1137 (ObjNum 1234 snap 269 seq_num 1234) dirty exists 2014-07-01 17:06:19,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: finishing write tid 1 to mira0437250-289 2014-07-01 17:06:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: finishing write tid 2 to mira0437250-289 2014-07-01 17:06:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: finishing write tid 3 to mira0437250-289 2014-07-01 17:06:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: finishing write tid 5 to mira0437250-289 2014-07-01 17:06:19,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2472: done (8 left) 2014-07-01 17:06:19,709.709 INFO:teuthology.task.rados.rados.0.mira043.stdout:2484: done (7 left) 2014-07-01 17:06:19,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2486: done (6 left) 2014-07-01 17:06:19,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2487: done (5 left) 2014-07-01 17:06:19,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2488: done (4 left) 2014-07-01 17:06:19,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2489: read oid 322 snap -1 2014-07-01 17:06:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: finishing write tid 6 to mira0437250-289 2014-07-01 17:06:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 833 (ObjNum 1232 snap 269 seq_num 1232) dirty exists 2014-07-01 17:06:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 1208 (ObjNum 372 snap 0 seq_num 372) dirty exists 2014-07-01 17:06:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: finishing write tid 1 to mira0437250-119 2014-07-01 17:06:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: finishing write tid 3 to mira0437250-119 2014-07-01 17:06:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: finishing write tid 4 to mira0437250-119 2014-07-01 17:06:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 119 v 914 (ObjNum 1235 snap 269 seq_num 1235) dirty exists 2014-07-01 17:06:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: finishing write tid 1 to mira0437250-480 2014-07-01 17:06:19,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: finishing write tid 2 to mira0437250-480 2014-07-01 17:06:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: finishing write tid 3 to mira0437250-480 2014-07-01 17:06:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: finishing write tid 5 to mira0437250-480 2014-07-01 17:06:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: finishing write tid 6 to mira0437250-480 2014-07-01 17:06:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 1049 (ObjNum 1231 snap 269 seq_num 1231) dirty exists 2014-07-01 17:06:19,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:2479: done (4 left) 2014-07-01 17:06:19,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2480: done (3 left) 2014-07-01 17:06:19,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:2481: done (2 left) 2014-07-01 17:06:19,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:2485: done (1 left) 2014-07-01 17:06:19,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2490: snap_create 2014-07-01 17:06:19,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2489: done (1 left) 2014-07-01 17:06:19,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2490: done (0 left) 2014-07-01 17:06:19,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: write oid 384 current snap is 271 2014-07-01 17:06:19,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: seq_num 1236 ranges {216342=1} 2014-07-01 17:06:19,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: writing mira0437250-384 from 216342 to 216343 tid 1 2014-07-01 17:06:19,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2492: snap_remove snap 270 2014-07-01 17:06:19,944.944 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2] out_osds: [3, 1] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:19,944.944 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:19,945.945 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 17:06:19,945.945 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 17:06:20,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 1 to mira0437250-384 2014-07-01 17:06:20,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 3 to mira0437250-384 2014-07-01 17:06:20,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: finishing write tid 4 to mira0437250-384 2014-07-01 17:06:20,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 1445 (ObjNum 1236 snap 271 seq_num 1236) dirty exists 2014-07-01 17:06:20,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: finishing write tid 1 to mira0435601-382 2014-07-01 17:06:20,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: finishing write tid 2 to mira0435601-382 2014-07-01 17:06:20,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: finishing write tid 3 to mira0435601-382 2014-07-01 17:06:20,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: finishing write tid 5 to mira0435601-382 2014-07-01 17:06:20,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: finishing write tid 6 to mira0435601-382 2014-07-01 17:06:20,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 131 (ObjNum 2102 snap 0 seq_num 2102) dirty exists 2014-07-01 17:06:20,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: finishing write tid 1 to mira0435601-461 2014-07-01 17:06:20,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: done (7 left) 2014-07-01 17:06:20,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3412: done (6 left) 2014-07-01 17:06:20,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3414: done (5 left) 2014-07-01 17:06:20,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3415: read oid 493 snap -1 2014-07-01 17:06:20,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: finishing write tid 2 to mira0435601-461 2014-07-01 17:06:20,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: read oid 313 snap -1 2014-07-01 17:06:20,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: read oid 113 snap -1 2014-07-01 17:06:20,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 307 current snap is 0 2014-07-01 17:06:20,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: seq_num 2105 ranges {0=774144,774144=532480,1306624=794624,2101248=278528} 2014-07-01 17:06:20,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: writing mira0435601-307 from 0 to 774144 tid 1 2014-07-01 17:06:20,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: writing mira0435601-307 from 774144 to 1306624 tid 2 2014-07-01 17:06:20,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: writing mira0435601-307 from 1306624 to 2101248 tid 3 2014-07-01 17:06:20,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: writing mira0435601-307 from 2101248 to 2379776 tid 4 2014-07-01 17:06:20,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: read oid 411 snap -1 2014-07-01 17:06:20,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: finishing write tid 3 to mira0435601-461 2014-07-01 17:06:20,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: expect (ObjNum 1727 snap 0 seq_num 1727) 2014-07-01 17:06:20,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2491: done (1 left) 2014-07-01 17:06:20,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:2492: done (0 left) 2014-07-01 17:06:20,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: write oid 389 current snap is 271 2014-07-01 17:06:20,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: seq_num 1237 ranges {409748=715092,1853069=156679} 2014-07-01 17:06:20,771.771 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 17:06:20,782.782 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 17:06:20,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: writing mira0437250-389 from 409748 to 1124840 tid 1 2014-07-01 17:06:20,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: writing mira0437250-389 from 1853069 to 2009748 tid 2 2014-07-01 17:06:20,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2494: delete oid 21 current snap is 271 2014-07-01 17:06:23,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: finishing write tid 5 to mira0435601-461 2014-07-01 17:06:23,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3415: expect (ObjNum 1821 snap 0 seq_num 1821) 2014-07-01 17:06:23,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 1 to mira0435601-307 2014-07-01 17:06:23,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 2 to mira0435601-307 2014-07-01 17:06:23,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 3 to mira0435601-307 2014-07-01 17:06:23,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 4 to mira0435601-307 2014-07-01 17:06:23,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 6 to mira0435601-307 2014-07-01 17:06:23,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: finishing write tid 7 to mira0435601-307 2014-07-01 17:06:23,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 256 (ObjNum 2105 snap 0 seq_num 2105) dirty exists 2014-07-01 17:06:23,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: finishing write tid 6 to mira0435601-461 2014-07-01 17:06:23,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 133 (ObjNum 2104 snap 0 seq_num 2104) dirty exists 2014-07-01 17:06:23,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: expect (ObjNum 1322 snap 0 seq_num 1322) 2014-07-01 17:06:24,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2494: done (1 left) 2014-07-01 17:06:24,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2495: delete oid 421 current snap is 271 2014-07-01 17:06:25,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2495: done (1 left) 2014-07-01 17:06:25,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2496: rollback oid 373 current snap is 271 2014-07-01 17:06:25,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 373 to 238 2014-07-01 17:06:25,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: write oid 413 current snap is 271 2014-07-01 17:06:25,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: seq_num 1240 ranges {509966=418025,1724298=728478,3206654=503312} 2014-07-01 17:06:25,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: writing mira0437250-413 from 509966 to 927991 tid 1 2014-07-01 17:06:25,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: writing mira0437250-413 from 1724298 to 2452776 tid 2 2014-07-01 17:06:25,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: writing mira0437250-413 from 3206654 to 3709966 tid 3 2014-07-01 17:06:25,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2498: snap_create 2014-07-01 17:06:25,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2498: done (3 left) 2014-07-01 17:06:25,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:2499: delete oid 202 current snap is 272 2014-07-01 17:06:25,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:2499: done (3 left) 2014-07-01 17:06:25,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2500: snap_remove snap 262 2014-07-01 17:06:25,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 1210 (ObjNum 372 snap 0 seq_num 372) dirty exists 2014-07-01 17:06:25,783.783 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:25,784.784 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:25,784.784 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2014-07-01 17:06:25,785.785 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 0' 2014-07-01 17:06:26,219.219 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: finishing write tid 1 to mira0437250-389 2014-07-01 17:06:26,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: finishing write tid 2 to mira0437250-389 2014-07-01 17:06:26,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: finishing write tid 4 to mira0437250-389 2014-07-01 17:06:26,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:2496: done (3 left) 2014-07-01 17:06:26,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2500: done (2 left) 2014-07-01 17:06:26,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2501: read oid 309 snap 258 2014-07-01 17:06:26,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: write oid 467 current snap is 272 2014-07-01 17:06:26,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: seq_num 1242 ranges {701449=514081,1952561=468868,3093775=7674} 2014-07-01 17:06:26,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: writing mira0437250-467 from 701449 to 1215530 tid 1 2014-07-01 17:06:26,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: writing mira0437250-467 from 1952561 to 2421429 tid 2 2014-07-01 17:06:26,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: writing mira0437250-467 from 3093775 to 3101449 tid 3 2014-07-01 17:06:26,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: write oid 6 current snap is 272 2014-07-01 17:06:26,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: seq_num 1243 ranges {645255=769534,1914246=574135,3178649=666606} 2014-07-01 17:06:26,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: writing mira0437250-6 from 645255 to 1414789 tid 1 2014-07-01 17:06:26,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: writing mira0437250-6 from 1914246 to 2488381 tid 2 2014-07-01 17:06:26,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: writing mira0437250-6 from 3178649 to 3845255 tid 3 2014-07-01 17:06:26,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: rollback oid 186 current snap is 272 2014-07-01 17:06:26,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 186 to 264 2014-07-01 17:06:26,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: finishing write tid 5 to mira0437250-389 2014-07-01 17:06:26,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 1178 (ObjNum 1237 snap 271 seq_num 1237) dirty exists 2014-07-01 17:06:26,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2501: expect (ObjNum 807 snap 130 seq_num 807) 2014-07-01 17:06:26,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: expect (ObjNum 1508 snap 0 seq_num 1508) 2014-07-01 17:06:27,291.291 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 0 (802) 2014-07-01 17:06:27,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 1023 (ObjNum 1208 snap 254 seq_num 1208) dirty exists 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 1 to mira0437250-467 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 2 to mira0437250-467 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 3 to mira0437250-467 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2493: done (5 left) 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2501: done (4 left) 2014-07-01 17:06:27,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2504: done (3 left) 2014-07-01 17:06:27,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: read oid 7 snap -1 2014-07-01 17:06:27,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: write oid 247 current snap is 272 2014-07-01 17:06:27,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: seq_num 1244 ranges {590084=1} 2014-07-01 17:06:28,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: writing mira0437250-247 from 590084 to 590085 tid 1 2014-07-01 17:06:28,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: read oid 488 snap -1 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2508: delete oid 242 current snap is 272 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: finishing write tid 1 to mira0437250-247 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: finishing write tid 3 to mira0437250-247 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: finishing write tid 4 to mira0437250-247 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 862 (ObjNum 1244 snap 272 seq_num 1244) dirty exists 2014-07-01 17:06:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: expect (ObjNum 1124 snap 228 seq_num 1124) 2014-07-01 17:06:29,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: finishing write tid 1 to mira0435601-452 2014-07-01 17:06:29,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: finishing write tid 2 to mira0435601-452 2014-07-01 17:06:29,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: finishing write tid 3 to mira0435601-452 2014-07-01 17:06:29,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: done (9 left) 2014-07-01 17:06:29,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: done (8 left) 2014-07-01 17:06:29,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3415: done (7 left) 2014-07-01 17:06:29,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: done (6 left) 2014-07-01 17:06:29,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: done (5 left) 2014-07-01 17:06:29,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: done (4 left) 2014-07-01 17:06:29,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:3420: delete oid 138 current snap is 0 2014-07-01 17:06:29,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3420: done (4 left) 2014-07-01 17:06:29,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 102 current snap is 0 2014-07-01 17:06:29,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: seq_num 2107 ranges {2809856=479232,3289088=401408} 2014-07-01 17:06:29,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: writing mira0435601-102 from 2809856 to 3289088 tid 1 2014-07-01 17:06:29,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: writing mira0435601-102 from 3289088 to 3690496 tid 2 2014-07-01 17:06:29,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 87 current snap is 0 2014-07-01 17:06:29,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: seq_num 2108 ranges {1261568=749568} 2014-07-01 17:06:29,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: writing mira0435601-87 from 1261568 to 2011136 tid 1 2014-07-01 17:06:29,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 101 current snap is 0 2014-07-01 17:06:29,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: seq_num 2109 ranges {3457024=622592} 2014-07-01 17:06:29,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: writing mira0435601-101 from 3457024 to 4079616 tid 1 2014-07-01 17:06:29,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3424: read oid 326 snap -1 2014-07-01 17:06:29,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: finishing write tid 1 to mira0435601-102 2014-07-01 17:06:29,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: finishing write tid 2 to mira0435601-102 2014-07-01 17:06:29,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: finishing write tid 4 to mira0435601-102 2014-07-01 17:06:29,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: finishing write tid 5 to mira0435601-102 2014-07-01 17:06:29,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 286 (ObjNum 2107 snap 0 seq_num 2107) dirty exists 2014-07-01 17:06:29,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: done (7 left) 2014-07-01 17:06:29,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3425: read oid 324 snap -1 2014-07-01 17:06:29,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 109 current snap is 0 2014-07-01 17:06:29,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: seq_num 2110 ranges {4931584=299008} 2014-07-01 17:06:29,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: writing mira0435601-109 from 4931584 to 5230592 tid 1 2014-07-01 17:06:29,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 446 current snap is 0 2014-07-01 17:06:29,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: seq_num 2111 ranges {0=167936} 2014-07-01 17:06:29,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: writing mira0435601-446 from 0 to 167936 tid 1 2014-07-01 17:06:29,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 74 current snap is 0 2014-07-01 17:06:29,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: seq_num 2112 ranges {4755456=40960} 2014-07-01 17:06:29,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: writing mira0435601-74 from 4755456 to 4796416 tid 1 2014-07-01 17:06:29,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 321 current snap is 0 2014-07-01 17:06:29,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: seq_num 2113 ranges {0=716800,716800=737280,1454080=688128,2142208=180224} 2014-07-01 17:06:29,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: writing mira0435601-321 from 0 to 716800 tid 1 2014-07-01 17:06:29,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: writing mira0435601-321 from 716800 to 1454080 tid 2 2014-07-01 17:06:29,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: writing mira0435601-321 from 1454080 to 2142208 tid 3 2014-07-01 17:06:29,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: writing mira0435601-321 from 2142208 to 2322432 tid 4 2014-07-01 17:06:29,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: read oid 40 snap -1 2014-07-01 17:06:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: finishing write tid 1 to mira0435601-446 2014-07-01 17:06:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: finishing write tid 3 to mira0435601-446 2014-07-01 17:06:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: finishing write tid 4 to mira0435601-446 2014-07-01 17:06:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 288 (ObjNum 2111 snap 0 seq_num 2111) dirty exists 2014-07-01 17:06:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3425: done (12 left) 2014-07-01 17:06:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: done (11 left) 2014-07-01 17:06:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 17:06:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: seq_num 2114 ranges {3104768=741376,3846144=401408} 2014-07-01 17:06:29,737.737 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: writing mira0435601-48 from 3104768 to 3846144 tid 1 2014-07-01 17:06:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: writing mira0435601-48 from 3846144 to 4247552 tid 2 2014-07-01 17:06:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 17:06:29,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: seq_num 2115 ranges {6844416=663552,7507968=503808,8011776=634880,8646656=65536} 2014-07-01 17:06:29,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: writing mira0435601-435 from 6844416 to 7507968 tid 1 2014-07-01 17:06:29,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: writing mira0435601-435 from 7507968 to 8011776 tid 2 2014-07-01 17:06:29,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: writing mira0435601-435 from 8011776 to 8646656 tid 3 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: writing mira0435601-435 from 8646656 to 8712192 tid 4 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 490 current snap is 0 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: seq_num 2116 ranges {4288512=536576,4825088=733184,5558272=458752,6017024=12288} 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: writing mira0435601-490 from 4288512 to 4825088 tid 1 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: writing mira0435601-490 from 4825088 to 5558272 tid 2 2014-07-01 17:06:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: writing mira0435601-490 from 5558272 to 6017024 tid 3 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: writing mira0435601-490 from 6017024 to 6029312 tid 4 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3434: read oid 202 snap -1 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: finishing write tid 1 to mira0435601-74 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: finishing write tid 3 to mira0435601-74 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: finishing write tid 4 to mira0435601-74 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 375 (ObjNum 2112 snap 0 seq_num 2112) dirty exists 2014-07-01 17:06:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: expect (ObjNum 1190 snap 0 seq_num 1190) 2014-07-01 17:06:30,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3434: expect (ObjNum 1836 snap 0 seq_num 1836) 2014-07-01 17:06:30,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: done (14 left) 2014-07-01 17:06:30,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: done (13 left) 2014-07-01 17:06:30,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3434: done (12 left) 2014-07-01 17:06:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 100 current snap is 0 2014-07-01 17:06:30,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: seq_num 2117 ranges {0=561152} 2014-07-01 17:06:30,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: writing mira0435601-100 from 0 to 561152 tid 1 2014-07-01 17:06:30,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: delete oid 262 current snap is 0 2014-07-01 17:06:30,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 5 to mira0437250-467 2014-07-01 17:06:30,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 1 to mira0437250-6 2014-07-01 17:06:30,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: finishing write tid 6 to mira0437250-467 2014-07-01 17:06:30,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 1282 (ObjNum 1242 snap 272 seq_num 1242) dirty exists 2014-07-01 17:06:30,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 2 to mira0437250-6 2014-07-01 17:06:30,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 3 to mira0437250-6 2014-07-01 17:06:30,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 5 to mira0437250-6 2014-07-01 17:06:30,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: finishing write tid 6 to mira0437250-6 2014-07-01 17:06:30,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 6 v 803 (ObjNum 1243 snap 272 seq_num 1243) dirty exists 2014-07-01 17:06:30,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: expect (ObjNum 649 snap 69 seq_num 649) 2014-07-01 17:06:30,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: finishing write tid 5 to mira0435601-452 2014-07-01 17:06:30,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 1 to mira0435601-321 2014-07-01 17:06:30,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: finishing write tid 1 to mira0435601-48 2014-07-01 17:06:30,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: finishing write tid 2 to mira0435601-48 2014-07-01 17:06:30,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 2 to mira0435601-321 2014-07-01 17:06:30,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: finishing write tid 1 to mira0435601-100 2014-07-01 17:06:30,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: finishing write tid 3 to mira0435601-100 2014-07-01 17:06:30,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: finishing write tid 4 to mira0435601-48 2014-07-01 17:06:30,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 3 to mira0435601-321 2014-07-01 17:06:30,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 4 to mira0435601-321 2014-07-01 17:06:30,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 6 to mira0435601-321 2014-07-01 17:06:32,303.303 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:32,304.304 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:32,304.304 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 17:06:32,305.305 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:06:32,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2502: done (6 left) 2014-07-01 17:06:32,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2503: done (5 left) 2014-07-01 17:06:32,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:2505: done (4 left) 2014-07-01 17:06:32,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2506: done (3 left) 2014-07-01 17:06:32,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2507: done (2 left) 2014-07-01 17:06:32,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2508: done (1 left) 2014-07-01 17:06:32,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2509: read oid 355 snap 259 2014-07-01 17:06:32,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: finishing write tid 1 to mira0437250-413 2014-07-01 17:06:32,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: finishing write tid 2 to mira0437250-413 2014-07-01 17:06:32,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: finishing write tid 3 to mira0437250-413 2014-07-01 17:06:32,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: write oid 137 current snap is 272 2014-07-01 17:06:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: seq_num 1246 ranges {746496=588819,1546495=1} 2014-07-01 17:06:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: writing mira0437250-137 from 746496 to 1335315 tid 1 2014-07-01 17:06:32,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: writing mira0437250-137 from 1546495 to 1546496 tid 2 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2511: rollback oid 479 current snap is 272 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 479 to 238 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: finishing write tid 5 to mira0437250-413 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: finishing write tid 6 to mira0437250-413 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 806 (ObjNum 1240 snap 271 seq_num 1240) dirty exists 2014-07-01 17:06:32,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:2497: done (3 left) 2014-07-01 17:06:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2509: done (2 left) 2014-07-01 17:06:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2512: read oid 121 snap -1 2014-07-01 17:06:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: write oid 310 current snap is 272 2014-07-01 17:06:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: seq_num 1247 ranges {691326=560623,1899606=391720} 2014-07-01 17:06:32,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: writing mira0437250-310 from 691326 to 1251949 tid 1 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: writing mira0437250-310 from 1899606 to 2291326 tid 2 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: read oid 74 snap -1 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2512: done (4 left) 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: write oid 71 current snap is 272 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: seq_num 1248 ranges {786556=416075,1861291=125265} 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: writing mira0437250-71 from 786556 to 1202631 tid 1 2014-07-01 17:06:32,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: writing mira0437250-71 from 1861291 to 1986556 tid 2 2014-07-01 17:06:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: read oid 377 snap -1 2014-07-01 17:06:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: snap_remove snap 269 2014-07-01 17:06:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: expect (ObjNum 683 snap 81 seq_num 683) 2014-07-01 17:06:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: finishing write tid 6 to mira0435601-452 2014-07-01 17:06:32,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 314 (ObjNum 2099 snap 0 seq_num 2099) dirty exists 2014-07-01 17:06:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: finishing write tid 4 to mira0435601-100 2014-07-01 17:06:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 100 v 280 (ObjNum 2117 snap 0 seq_num 2117) dirty exists 2014-07-01 17:06:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 1 to mira0435601-114 2014-07-01 17:06:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 2 to mira0435601-114 2014-07-01 17:06:32,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 4 to mira0435601-114 2014-07-01 17:06:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: finishing write tid 1 to mira0435601-87 2014-07-01 17:06:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: finishing write tid 3 to mira0435601-87 2014-07-01 17:06:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: finishing write tid 7 to mira0435601-321 2014-07-01 17:06:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 321 v 246 (ObjNum 2113 snap 0 seq_num 2113) dirty exists 2014-07-01 17:06:32,701.701 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: finishing write tid 4 to mira0435601-87 2014-07-01 17:06:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 87 v 226 (ObjNum 2108 snap 0 seq_num 2108) dirty exists 2014-07-01 17:06:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: finishing write tid 5 to mira0435601-48 2014-07-01 17:06:32,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 381 (ObjNum 2114 snap 0 seq_num 2114) dirty exists 2014-07-01 17:06:32,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 5 to mira0435601-114 2014-07-01 17:06:32,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 114 v 225 (ObjNum 2100 snap 0 seq_num 2100) dirty exists 2014-07-01 17:06:33,014.014 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:06:33,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: finishing write tid 1 to mira0435601-109 2014-07-01 17:06:33,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: finishing write tid 3 to mira0435601-109 2014-07-01 17:06:33,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: finishing write tid 1 to mira0437250-137 2014-07-01 17:06:33,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: finishing write tid 2 to mira0437250-137 2014-07-01 17:06:33,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: finishing write tid 4 to mira0437250-137 2014-07-01 17:06:33,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: finishing write tid 5 to mira0437250-137 2014-07-01 17:06:33,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 1455 (ObjNum 1246 snap 272 seq_num 1246) dirty exists 2014-07-01 17:06:33,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: expect (ObjNum 376 snap 0 seq_num 376) 2014-07-01 17:06:33,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 1 to mira0437250-310 2014-07-01 17:06:33,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 2 to mira0437250-310 2014-07-01 17:06:33,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 4 to mira0437250-310 2014-07-01 17:06:33,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 926 (ObjNum 591 snap 39 seq_num 591) dirty exists 2014-07-01 17:06:33,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2510: done (6 left) 2014-07-01 17:06:33,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2511: done (5 left) 2014-07-01 17:06:33,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2514: done (4 left) 2014-07-01 17:06:33,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2516: done (3 left) 2014-07-01 17:06:33,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2517: done (2 left) 2014-07-01 17:06:33,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2518: delete oid 178 current snap is 272 2014-07-01 17:06:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: finishing write tid 5 to mira0437250-310 2014-07-01 17:06:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 597 (ObjNum 1247 snap 272 seq_num 1247) dirty exists 2014-07-01 17:06:33,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3424: expect (ObjNum 1994 snap 0 seq_num 1994) 2014-07-01 17:06:34,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2513: done (2 left) 2014-07-01 17:06:34,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2518: done (1 left) 2014-07-01 17:06:34,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: read oid 459 snap 240 2014-07-01 17:06:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2520: rollback oid 173 current snap is 272 2014-07-01 17:06:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 173 to 263 2014-07-01 17:06:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: write oid 357 current snap is 272 2014-07-01 17:06:34,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: seq_num 1250 ranges {791543=512132,1770835=568350,2897883=761046,3991542=1} 2014-07-01 17:06:34,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0437250-357 from 791543 to 1303675 tid 1 2014-07-01 17:06:34,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0437250-357 from 1770835 to 2339185 tid 2 2014-07-01 17:06:34,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0437250-357 from 2897883 to 3658929 tid 3 2014-07-01 17:06:34,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: writing mira0437250-357 from 3991542 to 3991543 tid 4 2014-07-01 17:06:34,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:2522: snap_create 2014-07-01 17:06:34,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 0 (ObjNum 3 snap 0 seq_num 3) dirty dne 2014-07-01 17:06:34,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: expect (ObjNum 458 snap 0 seq_num 458) 2014-07-01 17:06:34,508.508 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: finishing write tid 1 to mira0437250-71 2014-07-01 17:06:34,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: finishing write tid 2 to mira0437250-71 2014-07-01 17:06:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: finishing write tid 4 to mira0437250-71 2014-07-01 17:06:34,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: finishing write tid 5 to mira0437250-71 2014-07-01 17:06:34,514.514 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 837 (ObjNum 1248 snap 272 seq_num 1248) dirty exists 2014-07-01 17:06:34,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 1 to mira0437250-357 2014-07-01 17:06:34,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 2 to mira0437250-357 2014-07-01 17:06:34,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 3 to mira0437250-357 2014-07-01 17:06:34,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 4 to mira0437250-357 2014-07-01 17:06:34,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 6 to mira0437250-357 2014-07-01 17:06:34,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: finishing write tid 7 to mira0437250-357 2014-07-01 17:06:34,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 1323 (ObjNum 1250 snap 272 seq_num 1250) dirty exists 2014-07-01 17:06:34,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: finishing write tid 4 to mira0435601-109 2014-07-01 17:06:34,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 237 (ObjNum 2110 snap 0 seq_num 2110) dirty exists 2014-07-01 17:06:34,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 1 to mira0435601-490 2014-07-01 17:06:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 2 to mira0435601-490 2014-07-01 17:06:34,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: finishing write tid 1 to mira0435601-101 2014-07-01 17:06:34,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 3 to mira0435601-490 2014-07-01 17:06:34,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: finishing write tid 3 to mira0435601-101 2014-07-01 17:06:34,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 4 to mira0435601-490 2014-07-01 17:06:34,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 6 to mira0435601-490 2014-07-01 17:06:34,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: expect (ObjNum 2028 snap 0 seq_num 2028) 2014-07-01 17:06:34,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2515: done (4 left) 2014-07-01 17:06:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2519: done (3 left) 2014-07-01 17:06:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2520: done (2 left) 2014-07-01 17:06:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2521: done (1 left) 2014-07-01 17:06:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2522: done (0 left) 2014-07-01 17:06:34,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: write oid 296 current snap is 273 2014-07-01 17:06:34,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: seq_num 1251 ranges {486773=484961,1733544=789969,3251557=435216} 2014-07-01 17:06:34,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0437250-296 from 486773 to 971734 tid 1 2014-07-01 17:06:34,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0437250-296 from 1733544 to 2523513 tid 2 2014-07-01 17:06:35,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: writing mira0437250-296 from 3251557 to 3686773 tid 3 2014-07-01 17:06:35,110.110 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: snap_remove snap 260 2014-07-01 17:06:35,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:2524: done (1 left) 2014-07-01 17:06:35,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: delete oid 195 current snap is 273 2014-07-01 17:06:36,231.231 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 56' 2014-07-01 17:06:36,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:2525: done (1 left) 2014-07-01 17:06:36,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2526: read oid 93 snap 194 2014-07-01 17:06:36,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: read oid 355 snap -1 2014-07-01 17:06:36,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:2528: delete oid 239 current snap is 273 2014-07-01 17:06:36,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:2527: done (3 left) 2014-07-01 17:06:36,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2528: done (2 left) 2014-07-01 17:06:36,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: read oid 413 snap -1 2014-07-01 17:06:36,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: read oid 215 snap -1 2014-07-01 17:06:36,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: read oid 39 snap -1 2014-07-01 17:06:36,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2532: snap_remove snap 250 2014-07-01 17:06:36,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2526: expect (ObjNum 696 snap 90 seq_num 696) 2014-07-01 17:06:36,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: expect (ObjNum 1240 snap 271 seq_num 1240) 2014-07-01 17:06:37,029.029 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 56 2014-07-01 17:06:39,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: expect (ObjNum 979 snap 181 seq_num 979) 2014-07-01 17:06:42,038.038 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1] out_osds: [3] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:42,038.038 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:42,039.039 INFO:teuthology.task.thrashosds.thrasher:Adding osd 3 2014-07-01 17:06:42,039.039 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 3' 2014-07-01 17:06:42,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 1 to mira0437250-296 2014-07-01 17:06:42,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 2 to mira0437250-296 2014-07-01 17:06:42,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 3 to mira0437250-296 2014-07-01 17:06:42,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 5 to mira0437250-296 2014-07-01 17:06:42,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: finishing write tid 6 to mira0437250-296 2014-07-01 17:06:42,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 1019 (ObjNum 1251 snap 273 seq_num 1251) dirty exists 2014-07-01 17:06:42,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2523: done (5 left) 2014-07-01 17:06:42,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2526: done (4 left) 2014-07-01 17:06:42,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2529: done (3 left) 2014-07-01 17:06:42,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2530: done (2 left) 2014-07-01 17:06:42,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2531: done (1 left) 2014-07-01 17:06:42,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:2532: done (0 left) 2014-07-01 17:06:42,606.606 INFO:teuthology.task.rados.rados.0.mira043.stdout:2533: read oid 205 snap -1 2014-07-01 17:06:42,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2534: snap_create 2014-07-01 17:06:43,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:2533: done (1 left) 2014-07-01 17:06:43,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2534: done (0 left) 2014-07-01 17:06:43,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2535: snap_remove snap 238 2014-07-01 17:06:43,328.328 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.3. 2014-07-01 17:06:43,355.355 INFO:teuthology.task.thrashosds.thrasher:Added osd 3 2014-07-01 17:06:44,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2535: done (0 left) 2014-07-01 17:06:44,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: read oid 329 snap 196 2014-07-01 17:06:44,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: write oid 13 current snap is 274 2014-07-01 17:06:44,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: seq_num 1254 ranges {588014=587894,1388013=1} 2014-07-01 17:06:44,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: writing mira0437250-13 from 588014 to 1175908 tid 1 2014-07-01 17:06:44,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: writing mira0437250-13 from 1388013 to 1388014 tid 2 2014-07-01 17:06:44,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: read oid 204 snap -1 2014-07-01 17:06:44,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: write oid 490 current snap is 274 2014-07-01 17:06:44,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: seq_num 1255 ranges {532844=559698,1718904=413940} 2014-07-01 17:06:44,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0437250-490 from 532844 to 1092542 tid 1 2014-07-01 17:06:44,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: writing mira0437250-490 from 1718904 to 2132844 tid 2 2014-07-01 17:06:44,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: read oid 439 snap -1 2014-07-01 17:06:44,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: read oid 485 snap 272 2014-07-01 17:06:44,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:2542: snap_remove snap 263 2014-07-01 17:06:44,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: expect (ObjNum 438 snap 0 seq_num 438) 2014-07-01 17:06:44,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: finishing write tid 7 to mira0435601-490 2014-07-01 17:06:44,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 141 (ObjNum 2116 snap 0 seq_num 2116) dirty exists 2014-07-01 17:06:44,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: finishing write tid 4 to mira0435601-101 2014-07-01 17:06:44,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 162 (ObjNum 2109 snap 0 seq_num 2109) dirty exists 2014-07-01 17:06:44,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 1 to mira0435601-435 2014-07-01 17:06:44,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 2 to mira0435601-435 2014-07-01 17:06:44,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 3 to mira0435601-435 2014-07-01 17:06:44,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 4 to mira0435601-435 2014-07-01 17:06:44,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 6 to mira0435601-435 2014-07-01 17:06:44,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: finishing write tid 7 to mira0435601-435 2014-07-01 17:06:44,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 375 (ObjNum 2115 snap 0 seq_num 2115) dirty exists 2014-07-01 17:06:44,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: finishing write tid 1 to mira0435601-27 2014-07-01 17:06:44,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: finishing write tid 3 to mira0435601-27 2014-07-01 17:06:44,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: finishing write tid 4 to mira0435601-27 2014-07-01 17:06:44,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 250 (ObjNum 2103 snap 0 seq_num 2103) dirty exists 2014-07-01 17:06:44,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: done (13 left) 2014-07-01 17:06:44,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: done (12 left) 2014-07-01 17:06:44,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: done (11 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: done (10 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: done (9 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: done (8 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3424: done (7 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: done (6 left) 2014-07-01 17:06:44,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: done (5 left) 2014-07-01 17:06:44,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: done (4 left) 2014-07-01 17:06:44,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: done (3 left) 2014-07-01 17:06:44,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: done (2 left) 2014-07-01 17:06:44,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: done (1 left) 2014-07-01 17:06:44,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: done (0 left) 2014-07-01 17:06:44,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3437: read oid 438 snap -1 2014-07-01 17:06:44,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 56 current snap is 0 2014-07-01 17:06:44,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: seq_num 2119 ranges {1949696=757760,2707456=401408} 2014-07-01 17:06:44,839.839 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: writing mira0435601-56 from 1949696 to 2707456 tid 1 2014-07-01 17:06:44,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: writing mira0435601-56 from 2707456 to 3108864 tid 2 2014-07-01 17:06:44,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3439: read oid 286 snap -1 2014-07-01 17:06:44,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 316 current snap is 0 2014-07-01 17:06:44,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: seq_num 2120 ranges {2732032=782336,3514368=716800,4231168=507904,4739072=380928} 2014-07-01 17:06:44,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0435601-316 from 2732032 to 3514368 tid 1 2014-07-01 17:06:44,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0435601-316 from 3514368 to 4231168 tid 2 2014-07-01 17:06:44,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0435601-316 from 4231168 to 4739072 tid 3 2014-07-01 17:06:44,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0435601-316 from 4739072 to 5120000 tid 4 2014-07-01 17:06:44,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3441: delete oid 180 current snap is 0 2014-07-01 17:06:45,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: finishing write tid 1 to mira0437250-13 2014-07-01 17:06:45,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: finishing write tid 2 to mira0437250-13 2014-07-01 17:06:45,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: finishing write tid 4 to mira0437250-13 2014-07-01 17:06:45,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: finishing write tid 5 to mira0437250-13 2014-07-01 17:06:45,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 13 v 781 (ObjNum 1254 snap 274 seq_num 1254) dirty exists 2014-07-01 17:06:45,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2537: done (6 left) 2014-07-01 17:06:45,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2538: done (5 left) 2014-07-01 17:06:45,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2540: done (4 left) 2014-07-01 17:06:45,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2541: done (3 left) 2014-07-01 17:06:45,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2542: done (2 left) 2014-07-01 17:06:45,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: write oid 53 current snap is 274 2014-07-01 17:06:45,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: seq_num 1256 ranges {628073=567598,1828072=1} 2014-07-01 17:06:45,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: writing mira0437250-53 from 628073 to 1195671 tid 1 2014-07-01 17:06:45,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: writing mira0437250-53 from 1828072 to 1828073 tid 2 2014-07-01 17:06:45,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: snap_remove snap 255 2014-07-01 17:06:46,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2544: done (3 left) 2014-07-01 17:06:46,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: write oid 127 current snap is 274 2014-07-01 17:06:46,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: seq_num 1257 ranges {571879=539403,1678368=596612,2571878=1} 2014-07-01 17:06:46,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0437250-127 from 571879 to 1111282 tid 1 2014-07-01 17:06:46,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0437250-127 from 1678368 to 2274980 tid 2 2014-07-01 17:06:46,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: writing mira0437250-127 from 2571878 to 2571879 tid 3 2014-07-01 17:06:46,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: write oid 286 current snap is 274 2014-07-01 17:06:46,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: seq_num 1258 ranges {632037=662631,2024180=666807,3252390=574730,3832036=1} 2014-07-01 17:06:46,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0437250-286 from 632037 to 1294668 tid 1 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0437250-286 from 2024180 to 2690987 tid 2 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0437250-286 from 3252390 to 3827120 tid 3 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: writing mira0437250-286 from 3832036 to 3832037 tid 4 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: delete oid 162 current snap is 274 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 1 to mira0437250-127 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: finishing write tid 1 to mira0437250-53 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 2 to mira0437250-127 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 3 to mira0437250-127 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 5 to mira0437250-127 2014-07-01 17:06:46,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: finishing write tid 6 to mira0437250-127 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 887 (ObjNum 1257 snap 274 seq_num 1257) dirty exists 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: finishing write tid 2 to mira0437250-53 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: finishing write tid 4 to mira0437250-53 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: finishing write tid 5 to mira0437250-53 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 927 (ObjNum 1256 snap 274 seq_num 1256) dirty exists 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2543: done (5 left) 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2545: done (4 left) 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2547: done (3 left) 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: write oid 31 current snap is 274 2014-07-01 17:06:46,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: seq_num 1260 ranges {273119=1} 2014-07-01 17:06:46,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: writing mira0437250-31 from 273119 to 273120 tid 1 2014-07-01 17:06:46,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2549: snap_create 2014-07-01 17:06:46,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: finishing write tid 1 to mira0437250-31 2014-07-01 17:06:46,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: finishing write tid 3 to mira0437250-31 2014-07-01 17:06:46,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 1 to mira0437250-286 2014-07-01 17:06:46,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 2 to mira0437250-286 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 3 to mira0437250-286 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 4 to mira0437250-286 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 6 to mira0437250-286 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: finishing write tid 4 to mira0437250-31 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 1326 (ObjNum 1260 snap 274 seq_num 1260) dirty exists 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: finishing write tid 7 to mira0437250-286 2014-07-01 17:06:46,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 286 v 950 (ObjNum 1258 snap 274 seq_num 1258) dirty exists 2014-07-01 17:06:46,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3441: done (4 left) 2014-07-01 17:06:46,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3442: read oid 152 snap -1 2014-07-01 17:06:46,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: read oid 422 snap -1 2014-07-01 17:06:46,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3444: delete oid 159 current snap is 0 2014-07-01 17:06:46,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3442: expect (ObjNum 1740 snap 0 seq_num 1740) 2014-07-01 17:06:47,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2546: done (4 left) 2014-07-01 17:06:47,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2548: done (3 left) 2014-07-01 17:06:47,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2549: done (2 left) 2014-07-01 17:06:47,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: snap_remove snap 194 2014-07-01 17:06:48,354.354 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:48,354.354 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:48,355.355 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:06:48,355.355 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:06:48,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:2550: done (2 left) 2014-07-01 17:06:48,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: rollback oid 312 current snap is 275 2014-07-01 17:06:48,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 312 to 256 2014-07-01 17:06:48,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: rollback oid 345 current snap is 275 2014-07-01 17:06:48,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 345 to 267 2014-07-01 17:06:48,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: read oid 377 snap -1 2014-07-01 17:06:48,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: write oid 445 current snap is 275 2014-07-01 17:06:48,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: seq_num 1261 ranges {616926=400000} 2014-07-01 17:06:48,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: writing mira0437250-445 from 616926 to 1016926 tid 1 2014-07-01 17:06:48,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:2555: snap_create 2014-07-01 17:06:48,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: expect (ObjNum 376 snap 0 seq_num 376) 2014-07-01 17:06:48,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 345 v 1246 (ObjNum 805 snap 130 seq_num 805) dirty exists 2014-07-01 17:06:48,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 1322 (ObjNum 1134 snap 232 seq_num 1134) dirty exists 2014-07-01 17:06:48,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 1 to mira0437250-445 2014-07-01 17:06:48,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 3 to mira0437250-445 2014-07-01 17:06:48,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: finishing write tid 4 to mira0437250-445 2014-07-01 17:06:48,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 1338 (ObjNum 1261 snap 275 seq_num 1261) dirty exists 2014-07-01 17:06:48,805.805 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: expect (ObjNum 1659 snap 0 seq_num 1659) 2014-07-01 17:06:49,087.087 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:06:49,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2551: done (6 left) 2014-07-01 17:06:49,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2552: done (5 left) 2014-07-01 17:06:49,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2553: done (4 left) 2014-07-01 17:06:49,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2554: done (3 left) 2014-07-01 17:06:49,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2555: done (2 left) 2014-07-01 17:06:49,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: write oid 80 current snap is 276 2014-07-01 17:06:49,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: seq_num 1262 ranges {713179=1} 2014-07-01 17:06:49,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: writing mira0437250-80 from 713179 to 713180 tid 1 2014-07-01 17:06:49,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: write oid 476 current snap is 276 2014-07-01 17:06:49,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: seq_num 1263 ranges {773338=557749,1971225=2113} 2014-07-01 17:06:49,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: writing mira0437250-476 from 773338 to 1331087 tid 1 2014-07-01 17:06:49,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: writing mira0437250-476 from 1971225 to 1973338 tid 2 2014-07-01 17:06:49,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: read oid 169 snap -1 2014-07-01 17:06:49,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 1 to mira0437250-80 2014-07-01 17:06:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: write oid 141 current snap is 276 2014-07-01 17:06:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: seq_num 1264 ranges {718167=565649,1818604=473450,2718166=1} 2014-07-01 17:06:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: writing mira0437250-141 from 718167 to 1283816 tid 1 2014-07-01 17:06:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: writing mira0437250-141 from 1818604 to 2292054 tid 2 2014-07-01 17:06:49,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: writing mira0437250-141 from 2718166 to 2718167 tid 3 2014-07-01 17:06:49,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: read oid 138 snap 246 2014-07-01 17:06:49,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 3 to mira0437250-80 2014-07-01 17:06:49,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 17:06:50,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: finishing write tid 4 to mira0437250-80 2014-07-01 17:06:50,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 983 (ObjNum 1262 snap 276 seq_num 1262) dirty exists 2014-07-01 17:06:50,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: expect (ObjNum 137 snap 0 seq_num 137) 2014-07-01 17:06:50,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3442: done (6 left) 2014-07-01 17:06:50,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: done (5 left) 2014-07-01 17:06:50,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3444: done (4 left) 2014-07-01 17:06:50,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 62 current snap is 0 2014-07-01 17:06:50,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: seq_num 2123 ranges {1949696=552960,2502656=401408} 2014-07-01 17:06:50,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: writing mira0435601-62 from 1949696 to 2502656 tid 1 2014-07-01 17:06:50,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: writing mira0435601-62 from 2502656 to 2904064 tid 2 2014-07-01 17:06:50,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:3446: delete oid 28 current snap is 0 2014-07-01 17:06:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:2556: done (6 left) 2014-07-01 17:06:50,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2558: done (5 left) 2014-07-01 17:06:50,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2560: done (4 left) 2014-07-01 17:06:50,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2561: delete oid 311 current snap is 276 2014-07-01 17:06:50,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2561: done (4 left) 2014-07-01 17:06:50,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: read oid 212 snap -1 2014-07-01 17:06:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: read oid 115 snap -1 2014-07-01 17:06:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: read oid 471 snap -1 2014-07-01 17:06:51,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: snap_create 2014-07-01 17:06:51,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: expect (ObjNum 1098 snap 220 seq_num 1098) 2014-07-01 17:06:51,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: expect (ObjNum 1198 snap 254 seq_num 1198) 2014-07-01 17:06:51,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:2562: done (7 left) 2014-07-01 17:06:51,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2563: done (6 left) 2014-07-01 17:06:51,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2565: done (5 left) 2014-07-01 17:06:51,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: read oid 403 snap -1 2014-07-01 17:06:51,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:2567: read oid 382 snap -1 2014-07-01 17:06:51,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: delete oid 149 current snap is 277 2014-07-01 17:06:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2568: done (7 left) 2014-07-01 17:06:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2569: snap_remove snap 254 2014-07-01 17:06:51,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: expect (ObjNum 402 snap 0 seq_num 402) 2014-07-01 17:06:51,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:2567: expect (ObjNum 1057 snap 211 seq_num 1057) 2014-07-01 17:06:52,352.352 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:06:52,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2566: done (7 left) 2014-07-01 17:06:52,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:2567: done (6 left) 2014-07-01 17:06:52,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2569: done (5 left) 2014-07-01 17:06:52,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: write oid 200 current snap is 277 2014-07-01 17:06:52,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: seq_num 1267 ranges {418384=1} 2014-07-01 17:06:52,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: writing mira0437250-200 from 418384 to 418385 tid 1 2014-07-01 17:06:52,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2571: snap_remove snap 224 2014-07-01 17:06:52,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 1 to mira0437250-200 2014-07-01 17:06:52,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 3 to mira0437250-200 2014-07-01 17:06:52,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: finishing write tid 4 to mira0437250-200 2014-07-01 17:06:52,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 200 v 893 (ObjNum 1267 snap 277 seq_num 1267) dirty exists 2014-07-01 17:06:53,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2570: done (6 left) 2014-07-01 17:06:53,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2571: done (5 left) 2014-07-01 17:06:53,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: read oid 169 snap 271 2014-07-01 17:06:53,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2573: read oid 74 snap -1 2014-07-01 17:06:53,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: read oid 123 snap -1 2014-07-01 17:06:53,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2575: read oid 131 snap 264 2014-07-01 17:06:53,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:2576: snap_remove snap 274 2014-07-01 17:06:53,442.442 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:06:53,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2575: expect (ObjNum 1017 snap 199 seq_num 1017) 2014-07-01 17:06:54,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:2573: expect (ObjNum 683 snap 81 seq_num 683) 2014-07-01 17:06:54,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: expect (ObjNum 996 snap 193 seq_num 996) 2014-07-01 17:06:56,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 17:06:56,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:3437: done (5 left) 2014-07-01 17:06:56,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:3446: done (4 left) 2014-07-01 17:06:56,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 435 current snap is 0 2014-07-01 17:06:56,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: seq_num 2125 ranges {8712192=745472,9457664=524288,9981952=278528} 2014-07-01 17:06:56,155.155 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: writing mira0435601-435 from 8712192 to 9457664 tid 1 2014-07-01 17:06:56,170.170 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: writing mira0435601-435 from 9457664 to 9981952 tid 2 2014-07-01 17:06:56,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: writing mira0435601-435 from 9981952 to 10260480 tid 3 2014-07-01 17:06:56,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: delete oid 421 current snap is 0 2014-07-01 17:06:56,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: finishing write tid 1 to mira0435601-56 2014-07-01 17:06:56,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: finishing write tid 2 to mira0435601-56 2014-07-01 17:06:56,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: finishing write tid 4 to mira0435601-56 2014-07-01 17:06:56,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 1 to mira0437250-141 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 2 to mira0437250-141 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 3 to mira0437250-141 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 5 to mira0437250-141 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: finishing write tid 6 to mira0437250-141 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 892 (ObjNum 1264 snap 276 seq_num 1264) dirty exists 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2559: done (9 left) 2014-07-01 17:06:56,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2572: done (8 left) 2014-07-01 17:06:56,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2573: done (7 left) 2014-07-01 17:06:56,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2574: done (6 left) 2014-07-01 17:06:56,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2575: done (5 left) 2014-07-01 17:06:56,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2576: done (4 left) 2014-07-01 17:06:56,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: snap_remove snap 267 2014-07-01 17:06:56,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: finishing write tid 1 to mira0435601-435 2014-07-01 17:06:56,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: finishing write tid 2 to mira0435601-435 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: finishing write tid 3 to mira0435601-435 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: finishing write tid 5 to mira0435601-435 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: done (5 left) 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: read oid 325 snap -1 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: read oid 474 snap -1 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3451: read oid 114 snap -1 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3452: delete oid 280 current snap is 0 2014-07-01 17:06:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: expect (ObjNum 2049 snap 0 seq_num 2049) 2014-07-01 17:06:58,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2577: done (4 left) 2014-07-01 17:06:58,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: snap_create 2014-07-01 17:06:58,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2578: done (4 left) 2014-07-01 17:06:58,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: snap_create 2014-07-01 17:06:58,458.458 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:06:58,458.458 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:06:58,458.458 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:06:58,459.459 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:06:58,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 1 to mira0437250-476 2014-07-01 17:06:58,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 2 to mira0437250-476 2014-07-01 17:06:58,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 4 to mira0437250-476 2014-07-01 17:06:58,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:3451: expect (ObjNum 2100 snap 0 seq_num 2100) 2014-07-01 17:06:59,052.052 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:06:59,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 1 to mira0437250-490 2014-07-01 17:06:59,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 2 to mira0437250-490 2014-07-01 17:06:59,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 4 to mira0437250-490 2014-07-01 17:06:59,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: finishing write tid 5 to mira0437250-490 2014-07-01 17:06:59,257.257 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 573 (ObjNum 1255 snap 274 seq_num 1255) dirty exists 2014-07-01 17:06:59,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: expect (ObjNum 328 snap 0 seq_num 328) 2014-07-01 17:07:01,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: finishing write tid 5 to mira0437250-476 2014-07-01 17:07:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 1238 (ObjNum 1263 snap 276 seq_num 1263) dirty exists 2014-07-01 17:07:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2536: done (4 left) 2014-07-01 17:07:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2539: done (3 left) 2014-07-01 17:07:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2557: done (2 left) 2014-07-01 17:07:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2579: done (1 left) 2014-07-01 17:07:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2580: rollback oid 311 current snap is 279 2014-07-01 17:07:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 311 to 271 2014-07-01 17:07:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: write oid 242 current snap is 279 2014-07-01 17:07:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: seq_num 1268 ranges {113613=1} 2014-07-01 17:07:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: writing mira0437250-242 from 113613 to 113614 tid 1 2014-07-01 17:07:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:2582: rollback oid 35 current snap is 279 2014-07-01 17:07:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 35 to 272 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: read oid 129 snap 273 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:2584: snap_create 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 1402 (ObjNum 857 snap 143 seq_num 857) dirty exists 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: expect (ObjNum 1205 snap 254 seq_num 1205) 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: finishing write tid 5 to mira0435601-56 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 136 (ObjNum 2119 snap 0 seq_num 2119) dirty exists 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: finishing write tid 6 to mira0435601-435 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 435 v 379 (ObjNum 2125 snap 0 seq_num 2125) dirty exists 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: finishing write tid 1 to mira0435601-62 2014-07-01 17:07:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: finishing write tid 2 to mira0435601-62 2014-07-01 17:07:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: finishing write tid 4 to mira0435601-62 2014-07-01 17:07:02,481.481 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 54' 2014-07-01 17:07:03,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 35 v 1288 (ObjNum 1224 snap 263 seq_num 1224) dirty exists 2014-07-01 17:07:03,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 1 to mira0435601-316 2014-07-01 17:07:03,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 2 to mira0435601-316 2014-07-01 17:07:03,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 3 to mira0435601-316 2014-07-01 17:07:03,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: finishing write tid 5 to mira0435601-62 2014-07-01 17:07:03,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 62 v 237 (ObjNum 2123 snap 0 seq_num 2123) dirty exists 2014-07-01 17:07:03,283.283 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 54 2014-07-01 17:07:03,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 4 to mira0435601-316 2014-07-01 17:07:03,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 6 to mira0435601-316 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2580: done (5 left) 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2582: done (4 left) 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2583: done (3 left) 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2584: done (2 left) 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: write oid 371 current snap is 280 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: seq_num 1269 ranges {458444=400000} 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: writing mira0437250-371 from 458444 to 858444 tid 1 2014-07-01 17:07:03,319.319 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: snap_create 2014-07-01 17:07:03,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: finishing write tid 1 to mira0437250-371 2014-07-01 17:07:03,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: finishing write tid 3 to mira0437250-371 2014-07-01 17:07:03,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: finishing write tid 4 to mira0437250-371 2014-07-01 17:07:03,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 1526 (ObjNum 1269 snap 280 seq_num 1269) dirty exists 2014-07-01 17:07:03,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 7 to mira0435601-316 2014-07-01 17:07:03,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 229 (ObjNum 2120 snap 0 seq_num 2120) dirty exists 2014-07-01 17:07:03,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: expect (ObjNum 1054 snap 210 seq_num 1054) 2014-07-01 17:07:03,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:3439: expect (ObjNum 2005 snap 0 seq_num 2005) 2014-07-01 17:07:05,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: finishing write tid 1 to mira0437250-242 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: finishing write tid 3 to mira0437250-242 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: finishing write tid 4 to mira0437250-242 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 856 (ObjNum 1268 snap 279 seq_num 1268) dirty exists 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2564: done (3 left) 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2581: done (2 left) 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2585: done (1 left) 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2586: done (0 left) 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: read oid 421 snap 258 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: rollback oid 235 current snap is 281 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 235 to 268 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: read oid 58 snap -1 2014-07-01 17:07:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:2590: read oid 243 snap -1 2014-07-01 17:07:05,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: snap_remove snap 265 2014-07-01 17:07:05,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 0 (ObjNum 1 snap 1 seq_num 39409344) dirty dne 2014-07-01 17:07:05,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: expect (ObjNum 983 snap 185 seq_num 983) 2014-07-01 17:07:05,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: expect (ObjNum 57 snap 0 seq_num 57) 2014-07-01 17:07:07,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2587: done (4 left) 2014-07-01 17:07:07,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2588: done (3 left) 2014-07-01 17:07:07,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2589: done (2 left) 2014-07-01 17:07:07,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2590: done (1 left) 2014-07-01 17:07:07,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2591: done (0 left) 2014-07-01 17:07:07,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2592: snap_remove snap 273 2014-07-01 17:07:08,292.292 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:07:08,292.292 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:08,292.292 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 17:07:08,292.292 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:07:08,292.292 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:07:08,292.292 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 17:07:13,744.744 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 3, 2, 4, 5] 2014-07-01 17:07:13,744.744 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:13,745.745 INFO:teuthology.task.thrashosds.thrasher:Killing osd 3, live_osds are [1, 0, 3, 2, 4, 5] 2014-07-01 17:07:13,746.746 DEBUG:teuthology.task.ceph.osd.3:waiting for process to exit 2014-07-01 17:07:13,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2592: done (0 left) 2014-07-01 17:07:13,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: write oid 107 current snap is 281 2014-07-01 17:07:13,845.845 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: seq_num 1270 ranges {403274=583995,1603273=1} 2014-07-01 17:07:13,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0437250-107 from 403274 to 987269 tid 1 2014-07-01 17:07:13,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: writing mira0437250-107 from 1603273 to 1603274 tid 2 2014-07-01 17:07:13,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: write oid 162 current snap is 281 2014-07-01 17:07:13,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: seq_num 1271 ranges {498504=708247,1298503=1} 2014-07-01 17:07:14,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: writing mira0437250-162 from 498504 to 1206751 tid 1 2014-07-01 17:07:14,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: writing mira0437250-162 from 1298503 to 1298504 tid 2 2014-07-01 17:07:14,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: write oid 105 current snap is 281 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: seq_num 1272 ranges {559685=563700,1920526=571908,2559684=1} 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0437250-105 from 559685 to 1123385 tid 1 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0437250-105 from 1920526 to 2492434 tid 2 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: writing mira0437250-105 from 2559684 to 2559685 tid 3 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2596: read oid 432 snap -1 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 1 to mira0437250-107 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 2 to mira0437250-107 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 4 to mira0437250-107 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: finishing write tid 5 to mira0437250-107 2014-07-01 17:07:14,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 107 v 956 (ObjNum 1270 snap 281 seq_num 1270) dirty exists 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 1 to mira0437250-162 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 2 to mira0437250-162 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 4 to mira0437250-162 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: finishing write tid 5 to mira0437250-162 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 162 v 1321 (ObjNum 1271 snap 281 seq_num 1271) dirty exists 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2593: done (3 left) 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2594: done (2 left) 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: read oid 178 snap -1 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2598: rollback oid 355 current snap is 281 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 355 to 276 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:2599: snap_remove snap 240 2014-07-01 17:07:14,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 0 (ObjNum 1 snap 1 seq_num 25409024) dirty dne 2014-07-01 17:07:14,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 1 to mira0437250-105 2014-07-01 17:07:14,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 2 to mira0437250-105 2014-07-01 17:07:14,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 3 to mira0437250-105 2014-07-01 17:07:14,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 5 to mira0437250-105 2014-07-01 17:07:14,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: finishing write tid 6 to mira0437250-105 2014-07-01 17:07:14,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 105 v 1287 (ObjNum 1272 snap 281 seq_num 1272) dirty exists 2014-07-01 17:07:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2595: done (4 left) 2014-07-01 17:07:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2597: done (3 left) 2014-07-01 17:07:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2598: done (2 left) 2014-07-01 17:07:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2599: done (1 left) 2014-07-01 17:07:14,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2600: snap_create 2014-07-01 17:07:15,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:2600: done (1 left) 2014-07-01 17:07:15,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: snap_remove snap 222 2014-07-01 17:07:16,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2601: done (1 left) 2014-07-01 17:07:16,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: read oid 229 snap -1 2014-07-01 17:07:16,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: read oid 144 snap -1 2014-07-01 17:07:16,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2604: read oid 272 snap -1 2014-07-01 17:07:16,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2605: snap_create 2014-07-01 17:07:16,936.936 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: expect (ObjNum 1082 snap 217 seq_num 1082) 2014-07-01 17:07:17,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: expect (ObjNum 1011 snap 197 seq_num 1011) 2014-07-01 17:07:17,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:2602: done (4 left) 2014-07-01 17:07:17,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2603: done (3 left) 2014-07-01 17:07:17,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2604: done (2 left) 2014-07-01 17:07:17,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2605: done (1 left) 2014-07-01 17:07:17,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:2606: snap_create 2014-07-01 17:07:18,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2606: done (1 left) 2014-07-01 17:07:18,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:2607: delete oid 314 current snap is 284 2014-07-01 17:07:18,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2607: done (1 left) 2014-07-01 17:07:18,948.948 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: snap_remove snap 264 2014-07-01 17:07:19,745.745 INFO:teuthology.task.ceph.osd.3:Stopped 2014-07-01 17:07:19,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2608: done (1 left) 2014-07-01 17:07:19,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: read oid 64 snap 156 2014-07-01 17:07:19,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: write oid 160 current snap is 284 2014-07-01 17:07:19,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: seq_num 1274 ranges {598721=659756,1712694=463722,2778300=220421} 2014-07-01 17:07:19,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: writing mira0437250-160 from 598721 to 1258477 tid 1 2014-07-01 17:07:19,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: writing mira0437250-160 from 1712694 to 2176416 tid 2 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: writing mira0437250-160 from 2778300 to 2998721 tid 3 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: read oid 30 snap 156 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2609: done (3 left) 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: write oid 451 current snap is 284 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: seq_num 1275 ranges {543551=782985,1943179=533917,3130950=453817,3743550=1} 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: writing mira0437250-451 from 543551 to 1326536 tid 1 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: writing mira0437250-451 from 1943179 to 2477096 tid 2 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: writing mira0437250-451 from 3130950 to 3584767 tid 3 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: writing mira0437250-451 from 3743550 to 3743551 tid 4 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: write oid 148 current snap is 284 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: seq_num 1276 ranges {603708=400000} 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: writing mira0437250-148 from 603708 to 1003708 tid 1 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2614: delete oid 283 current snap is 284 2014-07-01 17:07:20,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: finishing write tid 1 to mira0437250-160 2014-07-01 17:07:20,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: expect (ObjNum 29 snap 0 seq_num 29) 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: finishing write tid 2 to mira0437250-160 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: finishing write tid 3 to mira0437250-160 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: finishing write tid 5 to mira0437250-160 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: finishing write tid 6 to mira0437250-160 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 160 v 1327 (ObjNum 1274 snap 284 seq_num 1274) dirty exists 2014-07-01 17:07:20,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 1 to mira0437250-451 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 2 to mira0437250-451 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 3 to mira0437250-451 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2610: done (5 left) 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2611: done (4 left) 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2614: done (3 left) 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2615: rollback oid 427 current snap is 284 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 427 to 258 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 4 to mira0437250-451 2014-07-01 17:07:20,577.577 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 6 to mira0437250-451 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: finishing write tid 7 to mira0437250-451 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 451 v 1562 (ObjNum 1275 snap 284 seq_num 1275) dirty exists 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: finishing write tid 1 to mira0437250-148 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: finishing write tid 3 to mira0437250-148 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: finishing write tid 4 to mira0437250-148 2014-07-01 17:07:20,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 148 v 766 (ObjNum 1276 snap 284 seq_num 1276) dirty exists 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2612: done (3 left) 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2613: done (2 left) 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2616: read oid 104 snap 271 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2617: read oid 112 snap -1 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: delete oid 144 current snap is 284 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 1392 (ObjNum 426 snap 0 seq_num 426) dirty exists 2014-07-01 17:07:20,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2617: expect (ObjNum 111 snap 0 seq_num 111) 2014-07-01 17:07:21,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2615: done (4 left) 2014-07-01 17:07:21,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2617: done (3 left) 2014-07-01 17:07:21,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2618: done (2 left) 2014-07-01 17:07:21,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: snap_remove snap 256 2014-07-01 17:07:21,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:2619: done (2 left) 2014-07-01 17:07:21,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:2620: rollback oid 97 current snap is 284 2014-07-01 17:07:21,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 97 to 283 2014-07-01 17:07:21,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:2621: snap_remove snap 156 2014-07-01 17:07:22,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2621: done (3 left) 2014-07-01 17:07:22,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: delete oid 100 current snap is 284 2014-07-01 17:07:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2622: done (3 left) 2014-07-01 17:07:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: write oid 74 current snap is 284 2014-07-01 17:07:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: seq_num 1280 ranges {684851=714198,1884850=1} 2014-07-01 17:07:22,998.998 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: writing mira0437250-74 from 684851 to 1399049 tid 1 2014-07-01 17:07:23,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: writing mira0437250-74 from 1884850 to 1884851 tid 2 2014-07-01 17:07:23,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:2624: snap_remove snap 275 2014-07-01 17:07:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: finishing write tid 1 to mira0437250-74 2014-07-01 17:07:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: finishing write tid 2 to mira0437250-74 2014-07-01 17:07:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: finishing write tid 4 to mira0437250-74 2014-07-01 17:07:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: finishing write tid 5 to mira0437250-74 2014-07-01 17:07:23,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 1580 (ObjNum 1280 snap 284 seq_num 1280) dirty exists 2014-07-01 17:07:23,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2623: done (4 left) 2014-07-01 17:07:23,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2624: done (3 left) 2014-07-01 17:07:23,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: rollback oid 31 current snap is 284 2014-07-01 17:07:23,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 31 to 259 2014-07-01 17:07:23,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: rollback oid 408 current snap is 284 2014-07-01 17:07:23,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 408 to 258 2014-07-01 17:07:23,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: rollback oid 129 current snap is 284 2014-07-01 17:07:23,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 129 to 246 2014-07-01 17:07:23,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2628: rollback oid 479 current snap is 284 2014-07-01 17:07:23,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 479 to 246 2014-07-01 17:07:23,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: read oid 273 snap -1 2014-07-01 17:07:24,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2630: rollback oid 425 current snap is 284 2014-07-01 17:07:24,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 425 to 244 2014-07-01 17:07:24,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2631: delete oid 132 current snap is 284 2014-07-01 17:07:24,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: expect (ObjNum 1106 snap 222 seq_num 1106) 2014-07-01 17:07:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 950 (ObjNum 407 snap 0 seq_num 407) dirty exists 2014-07-01 17:07:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 479 v 996 (ObjNum 591 snap 39 seq_num 591) dirty exists 2014-07-01 17:07:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 1420 (ObjNum 30 snap 0 seq_num 30) dirty exists 2014-07-01 17:07:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 1528 (ObjNum 1092 snap 220 seq_num 1092) dirty exists 2014-07-01 17:07:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2625: done (9 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2626: done (8 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2627: done (7 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2628: done (6 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2629: done (5 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2631: done (4 left) 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2632: read oid 405 snap -1 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2633: rollback oid 306 current snap is 284 2014-07-01 17:07:24,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 306 to 244 2014-07-01 17:07:24,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2634: snap_create 2014-07-01 17:07:24,746.746 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:07:24,746.746 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:24,746.746 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 17:07:24,746.746 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:07:24,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2632: done (6 left) 2014-07-01 17:07:24,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2634: done (5 left) 2014-07-01 17:07:24,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: read oid 255 snap -1 2014-07-01 17:07:24,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: snap_create 2014-07-01 17:07:25,510.510 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:07:26,004.004 INFO:teuthology.task.rados.rados.0.mira043.stdout:2635: done (6 left) 2014-07-01 17:07:26,006.006 INFO:teuthology.task.rados.rados.0.mira043.stdout:2636: done (5 left) 2014-07-01 17:07:26,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2637: snap_remove snap 284 2014-07-01 17:07:26,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2637: done (5 left) 2014-07-01 17:07:27,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: snap_create 2014-07-01 17:07:28,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2638: done (5 left) 2014-07-01 17:07:28,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:2639: delete oid 181 current snap is 287 2014-07-01 17:07:28,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2639: done (5 left) 2014-07-01 17:07:28,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2640: snap_create 2014-07-01 17:07:29,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2640: done (5 left) 2014-07-01 17:07:29,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: write oid 299 current snap is 288 2014-07-01 17:07:29,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: seq_num 1283 ranges {784044=782060,2045100=633399,3335057=248987} 2014-07-01 17:07:29,051.051 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:07:29,052.052 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 66' 2014-07-01 17:07:29,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: writing mira0437250-299 from 784044 to 1566104 tid 1 2014-07-01 17:07:29,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: writing mira0437250-299 from 2045100 to 2678499 tid 2 2014-07-01 17:07:29,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: writing mira0437250-299 from 3335057 to 3584044 tid 3 2014-07-01 17:07:29,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2642: rollback oid 416 current snap is 288 2014-07-01 17:07:29,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 416 to 283 2014-07-01 17:07:29,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: snap_create 2014-07-01 17:07:29,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 416 v 0 (ObjNum 34561392 snap 0 seq_num 0) dirty dne 2014-07-01 17:07:30,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2642: done (7 left) 2014-07-01 17:07:30,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2643: done (6 left) 2014-07-01 17:07:30,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: write oid 123 current snap is 289 2014-07-01 17:07:30,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: seq_num 1284 ranges {328873=1} 2014-07-01 17:07:30,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: writing mira0437250-123 from 328873 to 328874 tid 1 2014-07-01 17:07:30,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2645: snap_remove snap 277 2014-07-01 17:07:30,059.059 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 66 2014-07-01 17:07:30,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: finishing write tid 1 to mira0437250-123 2014-07-01 17:07:30,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: finishing write tid 3 to mira0437250-123 2014-07-01 17:07:30,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: finishing write tid 4 to mira0437250-123 2014-07-01 17:07:30,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 1014 (ObjNum 1284 snap 289 seq_num 1284) dirty exists 2014-07-01 17:07:31,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2644: done (7 left) 2014-07-01 17:07:31,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2645: done (6 left) 2014-07-01 17:07:31,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: read oid 359 snap -1 2014-07-01 17:07:31,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:2647: delete oid 67 current snap is 289 2014-07-01 17:07:31,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: expect (ObjNum 1119 snap 226 seq_num 1119) 2014-07-01 17:07:32,001.001 INFO:teuthology.task.rados.rados.0.mira043.stdout:2646: done (7 left) 2014-07-01 17:07:32,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:2647: done (6 left) 2014-07-01 17:07:32,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: write oid 492 current snap is 289 2014-07-01 17:07:32,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: seq_num 1286 ranges {485286=733569,1285285=1} 2014-07-01 17:07:32,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: writing mira0437250-492 from 485286 to 1218855 tid 1 2014-07-01 17:07:32,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: writing mira0437250-492 from 1285285 to 1285286 tid 2 2014-07-01 17:07:32,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: read oid 499 snap -1 2014-07-01 17:07:32,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2650: snap_create 2014-07-01 17:07:32,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: finishing write tid 1 to mira0437250-492 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: finishing write tid 2 to mira0437250-492 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: finishing write tid 4 to mira0437250-492 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: finishing write tid 5 to mira0437250-492 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 1113 (ObjNum 1286 snap 289 seq_num 1286) dirty exists 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2648: done (8 left) 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2649: done (7 left) 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2650: done (6 left) 2014-07-01 17:07:32,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: snap_remove snap 196 2014-07-01 17:07:33,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:2651: done (6 left) 2014-07-01 17:07:33,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2652: read oid 25 snap -1 2014-07-01 17:07:33,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:2653: snap_remove snap 287 2014-07-01 17:07:34,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2653: done (7 left) 2014-07-01 17:07:34,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: read oid 378 snap -1 2014-07-01 17:07:34,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: write oid 115 current snap is 290 2014-07-01 17:07:34,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: seq_num 1287 ranges {430115=456797,1514108=516007} 2014-07-01 17:07:34,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: writing mira0437250-115 from 430115 to 886912 tid 1 2014-07-01 17:07:34,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: writing mira0437250-115 from 1514108 to 2030115 tid 2 2014-07-01 17:07:34,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: snap_create 2014-07-01 17:07:34,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 1 to mira0437250-115 2014-07-01 17:07:34,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 2 to mira0437250-115 2014-07-01 17:07:34,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 4 to mira0437250-115 2014-07-01 17:07:34,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: finishing write tid 5 to mira0437250-115 2014-07-01 17:07:34,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 874 (ObjNum 1287 snap 290 seq_num 1287) dirty exists 2014-07-01 17:07:35,077.077 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:07:35,077.077 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:35,077.077 INFO:teuthology.task.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2014-07-01 17:07:35,077.077 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 2 0' 2014-07-01 17:07:35,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2655: done (9 left) 2014-07-01 17:07:35,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2656: done (8 left) 2014-07-01 17:07:35,109.109 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: snap_create 2014-07-01 17:07:36,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2657: done (8 left) 2014-07-01 17:07:36,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: snap_remove snap 288 2014-07-01 17:07:36,139.139 INFO:teuthology.orchestra.run.mira079.stderr:set osd.2 primary-affinity to 0 (802) 2014-07-01 17:07:36,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 575 (ObjNum 305 snap 0 seq_num 305) dirty exists 2014-07-01 17:07:36,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: expect (ObjNum 686 snap 83 seq_num 686) 2014-07-01 17:07:36,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 425 v 858 (ObjNum 424 snap 0 seq_num 424) dirty exists 2014-07-01 17:07:36,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 0 (ObjNum 16554016 snap 0 seq_num 0) dirty dne 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: finishing write tid 1 to mira0437250-299 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: finishing write tid 2 to mira0437250-299 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: finishing write tid 3 to mira0437250-299 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: finishing write tid 5 to mira0437250-299 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: finishing write tid 6 to mira0437250-299 2014-07-01 17:07:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 897 (ObjNum 1283 snap 288 seq_num 1283) dirty exists 2014-07-01 17:07:36,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2596: expect (ObjNum 431 snap 0 seq_num 431) 2014-07-01 17:07:37,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:2652: expect (ObjNum 542 snap 15 seq_num 542) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2596: done (8 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2616: done (7 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2620: done (6 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2630: done (5 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2633: done (4 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2641: done (3 left) 2014-07-01 17:07:37,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:2652: done (2 left) 2014-07-01 17:07:37,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:2654: done (1 left) 2014-07-01 17:07:37,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:2658: done (0 left) 2014-07-01 17:07:37,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: read oid 349 snap -1 2014-07-01 17:07:37,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: write oid 417 current snap is 292 2014-07-01 17:07:37,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: seq_num 1288 ranges {525345=713273,1725344=1} 2014-07-01 17:07:37,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: writing mira0437250-417 from 525345 to 1238618 tid 1 2014-07-01 17:07:37,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: writing mira0437250-417 from 1725344 to 1725345 tid 2 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: read oid 174 snap -1 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: write oid 143 current snap is 292 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: seq_num 1289 ranges {469151=436502,1589925=441066,2469150=1} 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: writing mira0437250-143 from 469151 to 905653 tid 1 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: writing mira0437250-143 from 1589925 to 2030991 tid 2 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: writing mira0437250-143 from 2469150 to 2469151 tid 3 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2663: delete oid 244 current snap is 292 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: expect (ObjNum 697 snap 90 seq_num 697) 2014-07-01 17:07:37,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: expect (ObjNum 980 snap 183 seq_num 980) 2014-07-01 17:07:39,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: finishing write tid 1 to mira0437250-417 2014-07-01 17:07:39,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: finishing write tid 2 to mira0437250-417 2014-07-01 17:07:39,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: finishing write tid 4 to mira0437250-417 2014-07-01 17:07:39,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: finishing write tid 5 to mira0437250-417 2014-07-01 17:07:39,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 809 (ObjNum 1288 snap 292 seq_num 1288) dirty exists 2014-07-01 17:07:39,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 1 to mira0437250-143 2014-07-01 17:07:39,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 2 to mira0437250-143 2014-07-01 17:07:39,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 3 to mira0437250-143 2014-07-01 17:07:39,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 5 to mira0437250-143 2014-07-01 17:07:39,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: finishing write tid 6 to mira0437250-143 2014-07-01 17:07:39,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 814 (ObjNum 1289 snap 292 seq_num 1289) dirty exists 2014-07-01 17:07:39,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2659: done (4 left) 2014-07-01 17:07:39,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:2660: done (3 left) 2014-07-01 17:07:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2661: done (2 left) 2014-07-01 17:07:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2662: done (1 left) 2014-07-01 17:07:39,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:2663: done (0 left) 2014-07-01 17:07:39,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: read oid 127 snap -1 2014-07-01 17:07:39,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: read oid 156 snap -1 2014-07-01 17:07:39,168.168 INFO:teuthology.task.rados.rados.0.mira043.stdout:2666: delete oid 38 current snap is 292 2014-07-01 17:07:39,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2665: done (2 left) 2014-07-01 17:07:39,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2666: done (1 left) 2014-07-01 17:07:39,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: snap_create 2014-07-01 17:07:39,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: expect (ObjNum 1257 snap 274 seq_num 1257) 2014-07-01 17:07:40,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2664: done (1 left) 2014-07-01 17:07:40,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2667: done (0 left) 2014-07-01 17:07:40,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:2668: delete oid 326 current snap is 293 2014-07-01 17:07:40,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2668: done (0 left) 2014-07-01 17:07:40,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: read oid 257 snap 272 2014-07-01 17:07:40,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:2670: snap_remove snap 289 2014-07-01 17:07:40,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: expect (ObjNum 256 snap 0 seq_num 256) 2014-07-01 17:07:40,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: done (8 left) 2014-07-01 17:07:40,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3439: done (7 left) 2014-07-01 17:07:40,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: done (6 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: done (5 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: done (4 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: done (3 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: done (2 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3451: done (1 left) 2014-07-01 17:07:40,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3452: done (0 left) 2014-07-01 17:07:40,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 221 current snap is 0 2014-07-01 17:07:40,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: seq_num 2128 ranges {1814528=565248,2379776=401408} 2014-07-01 17:07:40,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: writing mira0435601-221 from 1814528 to 2379776 tid 1 2014-07-01 17:07:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: writing mira0435601-221 from 2379776 to 2781184 tid 2 2014-07-01 17:07:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3454: read oid 247 snap -1 2014-07-01 17:07:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 402 current snap is 0 2014-07-01 17:07:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: seq_num 2129 ranges {73728=593920,667648=507904,1175552=626688,1802240=434176,2236416=36864} 2014-07-01 17:07:40,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0435601-402 from 73728 to 667648 tid 1 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0435601-402 from 667648 to 1175552 tid 2 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0435601-402 from 1175552 to 1802240 tid 3 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0435601-402 from 1802240 to 2236416 tid 4 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0435601-402 from 2236416 to 2273280 tid 5 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 452 current snap is 0 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: seq_num 2130 ranges {2138112=622592,2760704=401408} 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: writing mira0435601-452 from 2138112 to 2760704 tid 1 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: writing mira0435601-452 from 2760704 to 3162112 tid 2 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 265 current snap is 0 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: seq_num 2131 ranges {3629056=737280,4366336=671744,5038080=532480} 2014-07-01 17:07:40,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2669: done (1 left) 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2670: done (0 left) 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: write oid 346 current snap is 293 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: seq_num 1293 ranges {514198=400000} 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: writing mira0437250-346 from 514198 to 914198 tid 1 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: snap_remove snap 283 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 1 to mira0437250-346 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 3 to mira0437250-346 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: finishing write tid 4 to mira0437250-346 2014-07-01 17:07:40,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 987 (ObjNum 1293 snap 293 seq_num 1293) dirty exists 2014-07-01 17:07:40,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0435601-265 from 3629056 to 4366336 tid 1 2014-07-01 17:07:40,684.684 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0435601-265 from 4366336 to 5038080 tid 2 2014-07-01 17:07:40,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0435601-265 from 5038080 to 5570560 tid 3 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3458: read oid 5 snap -1 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 1 to mira0435601-221 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 2 to mira0435601-221 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 4 to mira0435601-221 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 5 to mira0435601-221 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 369 (ObjNum 2128 snap 0 seq_num 2128) dirty exists 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 1 to mira0435601-402 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 2 to mira0435601-402 2014-07-01 17:07:40,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 3 to mira0435601-402 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 4 to mira0435601-402 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 5 to mira0435601-402 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 7 to mira0435601-402 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 8 to mira0435601-402 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 345 (ObjNum 2129 snap 0 seq_num 2129) dirty exists 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: finishing write tid 1 to mira0435601-452 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: finishing write tid 2 to mira0435601-452 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: finishing write tid 4 to mira0435601-452 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: finishing write tid 5 to mira0435601-452 2014-07-01 17:07:40,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 319 (ObjNum 2130 snap 0 seq_num 2130) dirty exists 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 1 to mira0435601-265 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: done (5 left) 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3454: done (4 left) 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: done (3 left) 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: done (2 left) 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: read oid 204 snap -1 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 256 current snap is 0 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: seq_num 2132 ranges {0=765952} 2014-07-01 17:07:40,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: writing mira0435601-256 from 0 to 765952 tid 1 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 52 current snap is 0 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: seq_num 2133 ranges {774144=442368} 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: writing mira0435601-52 from 774144 to 1216512 tid 1 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3462: read oid 224 snap -1 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 243 current snap is 0 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: seq_num 2134 ranges {4096000=311296} 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: writing mira0435601-243 from 4096000 to 4407296 tid 1 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 383 current snap is 0 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: seq_num 2135 ranges {2957312=184320} 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: writing mira0435601-383 from 2957312 to 3141632 tid 1 2014-07-01 17:07:40,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3465: delete oid 94 current snap is 0 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 2 to mira0435601-265 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 3 to mira0435601-265 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 5 to mira0435601-265 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 6 to mira0435601-265 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 265 v 254 (ObjNum 2131 snap 0 seq_num 2131) dirty exists 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: finishing write tid 1 to mira0435601-256 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: finishing write tid 3 to mira0435601-256 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: finishing write tid 4 to mira0435601-256 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 256 v 189 (ObjNum 2132 snap 0 seq_num 2132) dirty exists 2014-07-01 17:07:40,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 1 to mira0435601-52 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 3 to mira0435601-52 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 4 to mira0435601-52 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 52 v 383 (ObjNum 2133 snap 0 seq_num 2133) dirty exists 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: finishing write tid 1 to mira0435601-243 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: finishing write tid 3 to mira0435601-243 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: finishing write tid 4 to mira0435601-243 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 273 (ObjNum 2134 snap 0 seq_num 2134) dirty exists 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: done (8 left) 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3458: done (7 left) 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: done (6 left) 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: done (5 left) 2014-07-01 17:07:40,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: done (4 left) 2014-07-01 17:07:40,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3462: done (3 left) 2014-07-01 17:07:40,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: done (2 left) 2014-07-01 17:07:40,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3465: done (1 left) 2014-07-01 17:07:40,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3466: read oid 186 snap -1 2014-07-01 17:07:40,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 236 current snap is 0 2014-07-01 17:07:40,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: seq_num 2137 ranges {2117632=532480,2650112=712704,3362816=520192,3883008=372736} 2014-07-01 17:07:40,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0435601-236 from 2117632 to 2650112 tid 1 2014-07-01 17:07:40,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0435601-236 from 2650112 to 3362816 tid 2 2014-07-01 17:07:40,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0435601-236 from 3362816 to 3883008 tid 3 2014-07-01 17:07:40,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0435601-236 from 3883008 to 4255744 tid 4 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: seq_num 2138 ranges {4464640=405504,4870144=540672,5410816=794624,6205440=270336} 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: writing mira0435601-397 from 4464640 to 4870144 tid 1 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: writing mira0435601-397 from 4870144 to 5410816 tid 2 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: writing mira0435601-397 from 5410816 to 6205440 tid 3 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: writing mira0435601-397 from 6205440 to 6475776 tid 4 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 161 current snap is 0 2014-07-01 17:07:40,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: seq_num 2139 ranges {4583424=434176,5017600=401408} 2014-07-01 17:07:41,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: writing mira0435601-161 from 4583424 to 5017600 tid 1 2014-07-01 17:07:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: writing mira0435601-161 from 5017600 to 5419008 tid 2 2014-07-01 17:07:41,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:3470: read oid 434 snap -1 2014-07-01 17:07:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 1 to mira0435601-383 2014-07-01 17:07:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 3 to mira0435601-383 2014-07-01 17:07:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 4 to mira0435601-383 2014-07-01 17:07:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 133 (ObjNum 2135 snap 0 seq_num 2135) dirty exists 2014-07-01 17:07:41,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:3466: expect (ObjNum 1957 snap 0 seq_num 1957) 2014-07-01 17:07:41,150.150 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:07:41,150.150 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:41,150.150 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 17:07:41,150.150 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:07:41,150.150 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:07:41,150.150 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 17:07:41,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2671: done (1 left) 2014-07-01 17:07:41,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2672: done (0 left) 2014-07-01 17:07:41,603.603 INFO:teuthology.task.rados.rados.0.mira043.stdout:2673: read oid 396 snap -1 2014-07-01 17:07:41,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: write oid 29 current snap is 293 2014-07-01 17:07:41,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: seq_num 1294 ranges {726804=400000} 2014-07-01 17:07:41,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: writing mira0437250-29 from 726804 to 1126804 tid 1 2014-07-01 17:07:41,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2675: delete oid 52 current snap is 293 2014-07-01 17:07:41,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2675: done (2 left) 2014-07-01 17:07:41,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: write oid 27 current snap is 293 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: seq_num 1296 ranges {615439=711324,1978737=434256,2615438=1} 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: writing mira0437250-27 from 615439 to 1326763 tid 1 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: writing mira0437250-27 from 1978737 to 2412993 tid 2 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: writing mira0437250-27 from 2615438 to 2615439 tid 3 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2677: snap_remove snap 282 2014-07-01 17:07:41,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2673: expect (ObjNum 519 snap 10 seq_num 519) 2014-07-01 17:07:42,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: finishing write tid 1 to mira0437250-29 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: finishing write tid 3 to mira0437250-29 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: finishing write tid 4 to mira0437250-29 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 834 (ObjNum 1294 snap 293 seq_num 1294) dirty exists 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: finishing write tid 1 to mira0437250-27 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: finishing write tid 2 to mira0437250-27 2014-07-01 17:07:42,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: finishing write tid 3 to mira0437250-27 2014-07-01 17:07:42,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: finishing write tid 5 to mira0437250-27 2014-07-01 17:07:42,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: finishing write tid 6 to mira0437250-27 2014-07-01 17:07:42,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 1361 (ObjNum 1296 snap 293 seq_num 1296) dirty exists 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 1 to mira0435601-236 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 2 to mira0435601-236 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 3 to mira0435601-236 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 4 to mira0435601-236 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 6 to mira0435601-236 2014-07-01 17:07:42,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 7 to mira0435601-236 2014-07-01 17:07:42,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 203 (ObjNum 2137 snap 0 seq_num 2137) dirty exists 2014-07-01 17:07:42,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 1 to mira0435601-397 2014-07-01 17:07:42,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3471: read oid 325 snap -1 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: done (6 left) 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3466: done (5 left) 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: done (4 left) 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 422 current snap is 0 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: seq_num 2140 ranges {2736128=704512} 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: writing mira0435601-422 from 2736128 to 3440640 tid 1 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: read oid 73 snap -1 2014-07-01 17:07:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 2 to mira0435601-397 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 3 to mira0435601-397 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 4 to mira0435601-397 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 6 to mira0435601-397 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: finishing write tid 7 to mira0435601-397 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 169 (ObjNum 2138 snap 0 seq_num 2138) dirty exists 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 1 to mira0435601-161 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3470: expect (ObjNum 2016 snap 0 seq_num 2016) 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 2 to mira0435601-161 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 4 to mira0435601-161 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 5 to mira0435601-161 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 161 v 209 (ObjNum 2139 snap 0 seq_num 2139) dirty exists 2014-07-01 17:07:42,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: finishing write tid 1 to mira0435601-422 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: finishing write tid 3 to mira0435601-422 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: done (5 left) 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: done (4 left) 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3470: done (3 left) 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3471: done (2 left) 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: read oid 252 snap -1 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 59 current snap is 0 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: seq_num 2141 ranges {1855488=577536} 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: writing mira0435601-59 from 1855488 to 2433024 tid 1 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 185 current snap is 0 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: expect (ObjNum 1506 snap 0 seq_num 1506) 2014-07-01 17:07:42,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2673: done (3 left) 2014-07-01 17:07:42,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2674: done (2 left) 2014-07-01 17:07:42,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2676: done (1 left) 2014-07-01 17:07:42,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2677: done (0 left) 2014-07-01 17:07:42,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: snap_create 2014-07-01 17:07:43,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:2678: done (0 left) 2014-07-01 17:07:43,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: snap_remove snap 229 2014-07-01 17:07:44,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2679: done (0 left) 2014-07-01 17:07:44,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2680: rollback oid 126 current snap is 294 2014-07-01 17:07:44,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 126 to 268 2014-07-01 17:07:44,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: delete oid 254 current snap is 294 2014-07-01 17:07:44,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:07:44,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:2680: done (1 left) 2014-07-01 17:07:44,808.808 INFO:teuthology.task.rados.rados.0.mira043.stdout:2681: done (0 left) 2014-07-01 17:07:44,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:2682: delete oid 355 current snap is 294 2014-07-01 17:07:44,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2682: done (0 left) 2014-07-01 17:07:44,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:2683: delete oid 171 current snap is 294 2014-07-01 17:07:44,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2683: done (0 left) 2014-07-01 17:07:44,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2684: read oid 317 snap -1 2014-07-01 17:07:44,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:2685: snap_create 2014-07-01 17:07:45,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2684: done (1 left) 2014-07-01 17:07:45,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2685: done (0 left) 2014-07-01 17:07:45,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2686: read oid 143 snap -1 2014-07-01 17:07:45,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2687: read oid 358 snap 257 2014-07-01 17:07:45,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: rollback oid 455 current snap is 295 2014-07-01 17:07:45,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 455 to 281 2014-07-01 17:07:45,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: write oid 115 current snap is 295 2014-07-01 17:07:45,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: seq_num 1300 ranges {11909=1} 2014-07-01 17:07:45,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: writing mira0437250-115 from 11909 to 11910 tid 1 2014-07-01 17:07:45,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2690: snap_remove snap 293 2014-07-01 17:07:45,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 0 (ObjNum 3 snap 0 seq_num 1) dirty dne 2014-07-01 17:07:45,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 1 to mira0437250-115 2014-07-01 17:07:45,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:2686: expect (ObjNum 1289 snap 292 seq_num 1289) 2014-07-01 17:07:45,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: finishing write tid 1 to mira0435601-59 2014-07-01 17:07:45,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: finishing write tid 3 to mira0435601-59 2014-07-01 17:07:45,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: finishing write tid 4 to mira0435601-59 2014-07-01 17:07:45,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 59 v 287 (ObjNum 2141 snap 0 seq_num 2141) dirty exists 2014-07-01 17:07:45,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: seq_num 2142 ranges {1630208=253952} 2014-07-01 17:07:45,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: writing mira0435601-185 from 1630208 to 1884160 tid 1 2014-07-01 17:07:45,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: done (4 left) 2014-07-01 17:07:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: done (3 left) 2014-07-01 17:07:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: read oid 260 snap -1 2014-07-01 17:07:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 267 current snap is 0 2014-07-01 17:07:45,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: seq_num 2143 ranges {5292032=122880} 2014-07-01 17:07:46,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: writing mira0435601-267 from 5292032 to 5414912 tid 1 2014-07-01 17:07:46,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3479: read oid 196 snap -1 2014-07-01 17:07:46,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: expect (ObjNum 2051 snap 0 seq_num 2051) 2014-07-01 17:07:46,655.655 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:07:46,655.655 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:46,655.655 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_2 2014-07-01 17:07:46,655.655 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:07:47,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 3 to mira0437250-115 2014-07-01 17:07:47,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: finishing write tid 4 to mira0437250-115 2014-07-01 17:07:47,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 889 (ObjNum 1300 snap 295 seq_num 1300) dirty exists 2014-07-01 17:07:47,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:2687: expect (ObjNum 357 snap 0 seq_num 357) 2014-07-01 17:07:47,511.511 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:07:47,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:2686: done (4 left) 2014-07-01 17:07:47,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2687: done (3 left) 2014-07-01 17:07:47,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2688: done (2 left) 2014-07-01 17:07:47,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2689: done (1 left) 2014-07-01 17:07:47,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2690: done (0 left) 2014-07-01 17:07:47,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: delete oid 267 current snap is 295 2014-07-01 17:07:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2691: done (0 left) 2014-07-01 17:07:47,920.920 INFO:teuthology.task.rados.rados.0.mira043.stdout:2692: snap_remove snap 292 2014-07-01 17:07:48,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2692: done (0 left) 2014-07-01 17:07:48,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: write oid 471 current snap is 295 2014-07-01 17:07:48,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: seq_num 1302 ranges {700545=729670,1500544=1} 2014-07-01 17:07:48,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: writing mira0437250-471 from 700545 to 1430215 tid 1 2014-07-01 17:07:48,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: writing mira0437250-471 from 1500544 to 1500545 tid 2 2014-07-01 17:07:48,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: read oid 327 snap -1 2014-07-01 17:07:48,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: snap_create 2014-07-01 17:07:48,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 1 to mira0437250-471 2014-07-01 17:07:48,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 2 to mira0437250-471 2014-07-01 17:07:48,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 4 to mira0437250-471 2014-07-01 17:07:48,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: finishing write tid 5 to mira0437250-471 2014-07-01 17:07:48,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 872 (ObjNum 1302 snap 295 seq_num 1302) dirty exists 2014-07-01 17:07:49,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:2693: done (2 left) 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2694: done (1 left) 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2695: done (0 left) 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2696: rollback oid 156 current snap is 296 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 156 to 286 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2697: rollback oid 264 current snap is 296 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 264 to 281 2014-07-01 17:07:49,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:2698: snap_create 2014-07-01 17:07:49,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 156 v 0 (ObjNum 18854712 snap 0 seq_num 35043728) dirty dne 2014-07-01 17:07:49,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1440 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 17:07:50,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:2696: done (2 left) 2014-07-01 17:07:50,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:2697: done (1 left) 2014-07-01 17:07:50,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:2698: done (0 left) 2014-07-01 17:07:50,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: write oid 85 current snap is 297 2014-07-01 17:07:50,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: seq_num 1303 ranges {513151=453922,1435326=277825} 2014-07-01 17:07:50,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: writing mira0437250-85 from 513151 to 967073 tid 1 2014-07-01 17:07:50,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: writing mira0437250-85 from 1435326 to 1713151 tid 2 2014-07-01 17:07:50,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: rollback oid 371 current snap is 297 2014-07-01 17:07:50,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 371 to 244 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: write oid 281 current snap is 297 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: seq_num 1304 ranges {608381=709375,1408380=1} 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: writing mira0437250-281 from 608381 to 1317756 tid 1 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: writing mira0437250-281 from 1408380 to 1408381 tid 2 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2702: read oid 294 snap -1 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: finishing write tid 1 to mira0437250-85 2014-07-01 17:07:50,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: finishing write tid 2 to mira0437250-85 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: finishing write tid 4 to mira0437250-85 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: write oid 393 current snap is 297 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: seq_num 1305 ranges {553211=432603,1511143=603933,2153210=1} 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: writing mira0437250-393 from 553211 to 985814 tid 1 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: writing mira0437250-393 from 1511143 to 2115076 tid 2 2014-07-01 17:07:50,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: writing mira0437250-393 from 2153210 to 2153211 tid 3 2014-07-01 17:07:50,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2704: rollback oid 337 current snap is 297 2014-07-01 17:07:50,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 337 to 279 2014-07-01 17:07:50,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: finishing write tid 5 to mira0437250-85 2014-07-01 17:07:50,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 85 v 1383 (ObjNum 1303 snap 297 seq_num 1303) dirty exists 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 371 v 1644 (ObjNum 370 snap 0 seq_num 370) dirty exists 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: finishing write tid 1 to mira0437250-281 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2699: done (5 left) 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2700: done (4 left) 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2702: done (3 left) 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: write oid 384 current snap is 297 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: seq_num 1306 ranges {495992=405431,1472826=709200,2748519=147473} 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: writing mira0437250-384 from 495992 to 901423 tid 1 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: writing mira0437250-384 from 1472826 to 2182026 tid 2 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: writing mira0437250-384 from 2748519 to 2895992 tid 3 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: read oid 462 snap -1 2014-07-01 17:07:50,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: finishing write tid 2 to mira0437250-281 2014-07-01 17:07:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: finishing write tid 4 to mira0437250-281 2014-07-01 17:07:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: finishing write tid 5 to mira0437250-281 2014-07-01 17:07:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 1411 (ObjNum 1304 snap 297 seq_num 1304) dirty exists 2014-07-01 17:07:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2701: done (4 left) 2014-07-01 17:07:50,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: read oid 309 snap 266 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: finishing write tid 1 to mira0437250-393 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 1030 (ObjNum 834 snap 137 seq_num 834) dirty exists 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: finishing write tid 2 to mira0437250-393 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: finishing write tid 3 to mira0437250-393 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: finishing write tid 5 to mira0437250-393 2014-07-01 17:07:50,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: finishing write tid 6 to mira0437250-393 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 393 v 877 (ObjNum 1305 snap 297 seq_num 1305) dirty exists 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: finishing write tid 1 to mira0437250-384 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2703: done (4 left) 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2704: done (3 left) 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: write oid 331 current snap is 297 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: seq_num 1307 ranges {592246=528660,1585935=779395,2592245=1} 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: writing mira0437250-331 from 592246 to 1120906 tid 1 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: writing mira0437250-331 from 1585935 to 2365330 tid 2 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: writing mira0437250-331 from 2592245 to 2592246 tid 3 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: read oid 14 snap 279 2014-07-01 17:07:50,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: expect (ObjNum 461 snap 0 seq_num 461) 2014-07-01 17:07:51,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: expect (ObjNum 807 snap 130 seq_num 807) 2014-07-01 17:07:51,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: finishing write tid 4 to mira0435601-422 2014-07-01 17:07:51,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 281 (ObjNum 2140 snap 0 seq_num 2140) dirty exists 2014-07-01 17:07:51,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: finishing write tid 1 to mira0435601-185 2014-07-01 17:07:51,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: finishing write tid 3 to mira0435601-185 2014-07-01 17:07:51,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: done (5 left) 2014-07-01 17:07:51,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: done (4 left) 2014-07-01 17:07:51,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 343 current snap is 0 2014-07-01 17:07:51,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: seq_num 2144 ranges {1937408=798720,2736128=688128,3424256=585728,4009984=331776} 2014-07-01 17:07:51,430.430 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:07:51,431.431 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pg_num 66' 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: writing mira0435601-343 from 1937408 to 2736128 tid 1 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: writing mira0435601-343 from 2736128 to 3424256 tid 2 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: writing mira0435601-343 from 3424256 to 4009984 tid 3 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: writing mira0435601-343 from 4009984 to 4341760 tid 4 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3481: read oid 290 snap -1 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: finishing write tid 4 to mira0435601-185 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 164 (ObjNum 2142 snap 0 seq_num 2142) dirty exists 2014-07-01 17:07:51,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 17:07:52,703.703 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pg_num to 66 2014-07-01 17:07:52,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: finishing write tid 2 to mira0437250-384 2014-07-01 17:07:52,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: finishing write tid 3 to mira0437250-384 2014-07-01 17:07:52,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: finishing write tid 5 to mira0437250-384 2014-07-01 17:07:52,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: finishing write tid 6 to mira0437250-384 2014-07-01 17:07:52,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 1649 (ObjNum 1306 snap 297 seq_num 1306) dirty exists 2014-07-01 17:07:52,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 1 to mira0437250-331 2014-07-01 17:07:52,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 2 to mira0437250-331 2014-07-01 17:07:52,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2705: done (4 left) 2014-07-01 17:07:52,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:2706: done (3 left) 2014-07-01 17:07:52,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2707: done (2 left) 2014-07-01 17:07:52,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: rollback oid 454 current snap is 297 2014-07-01 17:07:52,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 454 to 276 2014-07-01 17:07:52,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 3 to mira0437250-331 2014-07-01 17:07:52,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 5 to mira0437250-331 2014-07-01 17:07:52,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: expect (ObjNum 945 snap 173 seq_num 945) 2014-07-01 17:07:53,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: finishing write tid 1 to mira0435601-267 2014-07-01 17:07:53,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: finishing write tid 3 to mira0435601-267 2014-07-01 17:07:53,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: finishing write tid 4 to mira0435601-267 2014-07-01 17:07:53,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 329 (ObjNum 2143 snap 0 seq_num 2143) dirty exists 2014-07-01 17:07:53,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3479: expect (ObjNum 1972 snap 0 seq_num 1972) 2014-07-01 17:07:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: finishing write tid 6 to mira0437250-331 2014-07-01 17:07:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 1004 (ObjNum 1307 snap 297 seq_num 1307) dirty exists 2014-07-01 17:07:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 1032 (ObjNum 1201 snap 254 seq_num 1201) dirty exists 2014-07-01 17:07:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2708: done (2 left) 2014-07-01 17:07:53,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2709: done (1 left) 2014-07-01 17:07:53,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2710: done (0 left) 2014-07-01 17:07:53,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2711: snap_remove snap 294 2014-07-01 17:07:53,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2711: done (0 left) 2014-07-01 17:07:53,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: delete oid 206 current snap is 297 2014-07-01 17:07:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2712: done (0 left) 2014-07-01 17:07:53,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: read oid 371 snap 268 2014-07-01 17:07:53,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: snap_remove snap 281 2014-07-01 17:07:53,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: expect (ObjNum 370 snap 0 seq_num 370) 2014-07-01 17:07:54,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:2713: done (1 left) 2014-07-01 17:07:54,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2714: done (0 left) 2014-07-01 17:07:54,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2715: read oid 216 snap 266 2014-07-01 17:07:54,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: write oid 204 current snap is 297 2014-07-01 17:07:54,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: seq_num 1309 ranges {597233=1} 2014-07-01 17:07:54,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: writing mira0437250-204 from 597233 to 597234 tid 1 2014-07-01 17:07:54,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2717: rollback oid 251 current snap is 297 2014-07-01 17:07:54,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 251 to 259 2014-07-01 17:07:54,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: read oid 377 snap -1 2014-07-01 17:07:54,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2719: delete oid 170 current snap is 297 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: expect (ObjNum 376 snap 0 seq_num 376) 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: finishing write tid 1 to mira0437250-204 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: finishing write tid 3 to mira0437250-204 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: finishing write tid 4 to mira0437250-204 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 1241 (ObjNum 1309 snap 297 seq_num 1309) dirty exists 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 763 (ObjNum 250 snap 0 seq_num 250) dirty exists 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2716: done (4 left) 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2717: done (3 left) 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2718: done (2 left) 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2719: done (1 left) 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: write oid 80 current snap is 297 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: seq_num 1311 ranges {637293=400000} 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: writing mira0437250-80 from 637293 to 1037293 tid 1 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: write oid 477 current snap is 297 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: seq_num 1312 ranges {698475=459873,1935027=363448} 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: writing mira0437250-477 from 698475 to 1158348 tid 1 2014-07-01 17:07:54,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: writing mira0437250-477 from 1935027 to 2298475 tid 2 2014-07-01 17:07:54,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: delete oid 238 current snap is 297 2014-07-01 17:07:54,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2715: expect (ObjNum 215 snap 0 seq_num 215) 2014-07-01 17:07:55,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: finishing write tid 1 to mira0437250-80 2014-07-01 17:07:55,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: finishing write tid 3 to mira0437250-80 2014-07-01 17:07:55,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: finishing write tid 4 to mira0437250-80 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 1039 (ObjNum 1311 snap 297 seq_num 1311) dirty exists 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: finishing write tid 1 to mira0437250-477 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: finishing write tid 2 to mira0437250-477 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: finishing write tid 4 to mira0437250-477 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: finishing write tid 5 to mira0437250-477 2014-07-01 17:07:55,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 1665 (ObjNum 1312 snap 297 seq_num 1312) dirty exists 2014-07-01 17:07:55,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2715: done (3 left) 2014-07-01 17:07:55,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2720: done (2 left) 2014-07-01 17:07:55,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2721: done (1 left) 2014-07-01 17:07:55,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2722: done (0 left) 2014-07-01 17:07:55,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:2723: snap_create 2014-07-01 17:07:55,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2723: done (0 left) 2014-07-01 17:07:55,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: write oid 446 current snap is 298 2014-07-01 17:07:55,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: seq_num 1314 ranges {737510=404506,1575771=524010,2552627=184883} 2014-07-01 17:07:55,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: writing mira0437250-446 from 737510 to 1142016 tid 1 2014-07-01 17:07:55,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: writing mira0437250-446 from 1575771 to 2099781 tid 2 2014-07-01 17:07:55,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: writing mira0437250-446 from 2552627 to 2737510 tid 3 2014-07-01 17:07:55,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: read oid 202 snap -1 2014-07-01 17:07:55,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 1 to mira0437250-446 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: snap_create 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 2 to mira0437250-446 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 3 to mira0437250-446 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 5 to mira0437250-446 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: finishing write tid 6 to mira0437250-446 2014-07-01 17:07:55,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 1418 (ObjNum 1314 snap 298 seq_num 1314) dirty exists 2014-07-01 17:07:56,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 1 to mira0435601-343 2014-07-01 17:07:56,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 2 to mira0435601-343 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 3 to mira0435601-343 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 4 to mira0435601-343 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 6 to mira0435601-343 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: finishing write tid 7 to mira0435601-343 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 342 (ObjNum 2144 snap 0 seq_num 2144) dirty exists 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: done (5 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: done (4 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: done (3 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3479: done (2 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: done (1 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3481: done (0 left) 2014-07-01 17:07:56,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3482: delete oid 400 current snap is 0 2014-07-01 17:07:56,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3482: done (0 left) 2014-07-01 17:07:56,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3483: read oid 300 snap -1 2014-07-01 17:07:56,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3484: read oid 296 snap -1 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: read oid 104 snap -1 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 408 current snap is 0 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: seq_num 2146 ranges {5935104=745472,6680576=749568,7430144=454656} 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: writing mira0435601-408 from 5935104 to 6680576 tid 1 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: writing mira0435601-408 from 6680576 to 7430144 tid 2 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: writing mira0435601-408 from 7430144 to 7884800 tid 3 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 455 current snap is 0 2014-07-01 17:07:56,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: seq_num 2147 ranges {4898816=774144} 2014-07-01 17:07:56,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: writing mira0435601-455 from 4898816 to 5672960 tid 1 2014-07-01 17:07:56,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 66 current snap is 0 2014-07-01 17:07:56,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: seq_num 2148 ranges {4624384=643072} 2014-07-01 17:07:56,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: writing mira0435601-66 from 4624384 to 5267456 tid 1 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: read oid 54 snap -1 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 377 current snap is 0 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: seq_num 2149 ranges {5152768=516096} 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: writing mira0435601-377 from 5152768 to 5668864 tid 1 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 170 current snap is 0 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: seq_num 2150 ranges {4923392=544768,5468160=729088,6197248=475136} 2014-07-01 17:07:56,579.579 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: writing mira0435601-170 from 4923392 to 5468160 tid 1 2014-07-01 17:07:56,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: writing mira0435601-170 from 5468160 to 6197248 tid 2 2014-07-01 17:07:56,601.601 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: writing mira0435601-170 from 6197248 to 6672384 tid 3 2014-07-01 17:07:56,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:3492: read oid 457 snap -1 2014-07-01 17:07:56,602.602 INFO:teuthology.task.rados.rados.0.mira043.stdout:3484: expect (ObjNum 1808 snap 0 seq_num 1808) 2014-07-01 17:07:56,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: expect (ObjNum 1649 snap 0 seq_num 1649) 2014-07-01 17:07:56,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:2724: done (2 left) 2014-07-01 17:07:56,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2725: done (1 left) 2014-07-01 17:07:56,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2726: done (0 left) 2014-07-01 17:07:56,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:2727: snap_create 2014-07-01 17:07:56,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3483: expect (ObjNum 1732 snap 0 seq_num 1732) 2014-07-01 17:07:57,713.713 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:07:57,713.713 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:07:57,713.713 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:07:57,713.713 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:07:57,713.713 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:07:57,713.713 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:07:57,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2727: done (0 left) 2014-07-01 17:07:57,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: write oid 411 current snap is 300 2014-07-01 17:07:57,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: seq_num 1315 ranges {682340=527735,1805232=726429,3152829=329511} 2014-07-01 17:07:57,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: writing mira0437250-411 from 682340 to 1210075 tid 1 2014-07-01 17:07:57,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: writing mira0437250-411 from 1805232 to 2531661 tid 2 2014-07-01 17:07:57,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: writing mira0437250-411 from 3152829 to 3482340 tid 3 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2729: rollback oid 132 current snap is 300 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 132 to 278 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: finishing write tid 1 to mira0437250-411 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: finishing write tid 2 to mira0437250-411 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2730: snap_remove snap 271 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: finishing write tid 3 to mira0437250-411 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: finishing write tid 5 to mira0437250-411 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: finishing write tid 6 to mira0437250-411 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 882 (ObjNum 1315 snap 300 seq_num 1315) dirty exists 2014-07-01 17:07:57,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 132 v 1014 (ObjNum 131 snap 0 seq_num 131) dirty exists 2014-07-01 17:07:58,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2728: done (2 left) 2014-07-01 17:07:58,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2729: done (1 left) 2014-07-01 17:07:58,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2730: done (0 left) 2014-07-01 17:07:58,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2731: delete oid 36 current snap is 300 2014-07-01 17:07:58,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2731: done (0 left) 2014-07-01 17:07:58,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: snap_remove snap 296 2014-07-01 17:07:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2732: done (0 left) 2014-07-01 17:07:59,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: read oid 284 snap -1 2014-07-01 17:07:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: write oid 46 current snap is 300 2014-07-01 17:07:59,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: seq_num 1317 ranges {438750=1} 2014-07-01 17:07:59,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: writing mira0437250-46 from 438750 to 438751 tid 1 2014-07-01 17:07:59,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: read oid 488 snap 298 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2736: snap_create 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: expect (ObjNum 649 snap 69 seq_num 649) 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: finishing write tid 1 to mira0435601-408 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: finishing write tid 2 to mira0435601-408 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: finishing write tid 3 to mira0435601-408 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: finishing write tid 5 to mira0435601-408 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3483: done (9 left) 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3484: done (8 left) 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: done (7 left) 2014-07-01 17:07:59,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 177 current snap is 0 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: seq_num 2151 ranges {8355840=61440} 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: writing mira0435601-177 from 8355840 to 8417280 tid 1 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3494: delete oid 154 current snap is 0 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: finishing write tid 6 to mira0435601-408 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 334 (ObjNum 2146 snap 0 seq_num 2146) dirty exists 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 1 to mira0435601-455 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 3 to mira0435601-455 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 4 to mira0435601-455 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 336 (ObjNum 2147 snap 0 seq_num 2147) dirty exists 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: finishing write tid 1 to mira0435601-66 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: finishing write tid 3 to mira0435601-66 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: finishing write tid 4 to mira0435601-66 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 238 (ObjNum 2148 snap 0 seq_num 2148) dirty exists 2014-07-01 17:07:59,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: expect (ObjNum 1552 snap 0 seq_num 1552) 2014-07-01 17:08:00,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: expect (ObjNum 781 snap 124 seq_num 781) 2014-07-01 17:08:01,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:2733: done (3 left) 2014-07-01 17:08:01,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:2735: done (2 left) 2014-07-01 17:08:01,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:2736: done (1 left) 2014-07-01 17:08:01,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:2737: snap_create 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: finishing write tid 1 to mira0435601-377 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: finishing write tid 3 to mira0435601-377 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: finishing write tid 4 to mira0435601-377 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 282 (ObjNum 2149 snap 0 seq_num 2149) dirty exists 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 1 to mira0435601-170 2014-07-01 17:08:01,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 2 to mira0435601-170 2014-07-01 17:08:01,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 3 to mira0435601-170 2014-07-01 17:08:01,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 5 to mira0435601-170 2014-07-01 17:08:01,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 6 to mira0435601-170 2014-07-01 17:08:01,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 229 (ObjNum 2150 snap 0 seq_num 2150) dirty exists 2014-07-01 17:08:01,679.679 INFO:teuthology.task.rados.rados.0.mira043.stdout:3492: expect (ObjNum 1911 snap 0 seq_num 1911) 2014-07-01 17:08:01,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:2737: done (1 left) 2014-07-01 17:08:01,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: snap_remove snap 297 2014-07-01 17:08:01,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: finishing write tid 1 to mira0437250-46 2014-07-01 17:08:01,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: finishing write tid 3 to mira0437250-46 2014-07-01 17:08:01,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: finishing write tid 4 to mira0437250-46 2014-07-01 17:08:01,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 730 (ObjNum 1317 snap 300 seq_num 1317) dirty exists 2014-07-01 17:08:02,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2734: done (1 left) 2014-07-01 17:08:02,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2738: done (0 left) 2014-07-01 17:08:02,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:2739: snap_remove snap 298 2014-07-01 17:08:03,142.142 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [3] live_osds: [1, 0, 2, 4, 5] 2014-07-01 17:08:03,142.142 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:03,142.142 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 3 2014-07-01 17:08:03,142.142 INFO:teuthology.task.ceph.osd.3:Restarting daemon 2014-07-01 17:08:03,142.142 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 3' 2014-07-01 17:08:03,511.511 INFO:teuthology.task.ceph.osd.3:Started 2014-07-01 17:08:03,511.511 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_ops_in_flight' 2014-07-01 17:08:03,570.570 INFO:teuthology.task.ceph.osd.3.mira028.stdout:starting osd.3 at :/0 osd_data /var/lib/ceph/osd/ceph-3 /var/lib/ceph/osd/ceph-3/journal 2014-07-01 17:08:03,680.680 INFO:teuthology.task.ceph.osd.3.mira028.stderr:2014-07-01 08:08:03.589826 7fb53cf08780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 17:08:03,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:2739: done (0 left) 2014-07-01 17:08:03,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: write oid 475 current snap is 302 2014-07-01 17:08:03,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: seq_num 1318 ranges {135004=1} 2014-07-01 17:08:03,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: writing mira0437250-475 from 135004 to 135005 tid 1 2014-07-01 17:08:03,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: write oid 236 current snap is 302 2014-07-01 17:08:03,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: seq_num 1319 ranges {727387=602472,1922817=4570} 2014-07-01 17:08:03,934.934 INFO:teuthology.orchestra.run.mira028.stderr:no valid command found; 10 closest matches: 2014-07-01 17:08:03,935.935 INFO:teuthology.orchestra.run.mira028.stderr:config show 2014-07-01 17:08:03,935.935 INFO:teuthology.orchestra.run.mira028.stderr:help 2014-07-01 17:08:03,935.935 INFO:teuthology.orchestra.run.mira028.stderr:log dump 2014-07-01 17:08:03,935.935 INFO:teuthology.orchestra.run.mira028.stderr:get_command_descriptions 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:git_version 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:config set [...] 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:version 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:2 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:config get 2014-07-01 17:08:03,936.936 INFO:teuthology.orchestra.run.mira028.stderr:0 2014-07-01 17:08:03,937.937 INFO:teuthology.orchestra.run.mira028.stderr:admin_socket: invalid command 2014-07-01 17:08:03,940.940 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 3, ['dump_ops_in_flight'] 2014-07-01 17:08:04,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: writing mira0437250-236 from 727387 to 1329859 tid 1 2014-07-01 17:08:04,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: writing mira0437250-236 from 1922817 to 1927387 tid 2 2014-07-01 17:08:04,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2742: snap_create 2014-07-01 17:08:04,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 1 to mira0437250-475 2014-07-01 17:08:04,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 3 to mira0437250-475 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: finishing write tid 4 to mira0437250-475 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 1229 (ObjNum 1318 snap 302 seq_num 1318) dirty exists 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 1 to mira0437250-236 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 2 to mira0437250-236 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 4 to mira0437250-236 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: finishing write tid 5 to mira0437250-236 2014-07-01 17:08:04,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 731 (ObjNum 1319 snap 302 seq_num 1319) dirty exists 2014-07-01 17:08:04,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: finishing write tid 1 to mira0435601-177 2014-07-01 17:08:04,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: finishing write tid 3 to mira0435601-177 2014-07-01 17:08:04,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: finishing write tid 4 to mira0435601-177 2014-07-01 17:08:04,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 177 v 164 (ObjNum 2151 snap 0 seq_num 2151) dirty exists 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: done (8 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: done (7 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: done (6 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: done (5 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: done (4 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: done (3 left) 2014-07-01 17:08:04,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3492: done (2 left) 2014-07-01 17:08:04,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: done (1 left) 2014-07-01 17:08:04,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3494: done (0 left) 2014-07-01 17:08:04,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3495: read oid 310 snap -1 2014-07-01 17:08:04,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3496: read oid 349 snap -1 2014-07-01 17:08:04,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 153 current snap is 0 2014-07-01 17:08:04,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: seq_num 2153 ranges {4902912=765952,5668864=401408} 2014-07-01 17:08:04,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: writing mira0435601-153 from 4902912 to 5668864 tid 1 2014-07-01 17:08:04,662.662 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: writing mira0435601-153 from 5668864 to 6070272 tid 2 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: seq_num 2154 ranges {6676480=634880,7311360=401408} 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: writing mira0435601-11 from 6676480 to 7311360 tid 1 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: writing mira0435601-11 from 7311360 to 7712768 tid 2 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 125 current snap is 0 2014-07-01 17:08:04,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: seq_num 2155 ranges {3915776=557056,4472832=647168,5120000=557056} 2014-07-01 17:08:04,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: writing mira0435601-125 from 3915776 to 4472832 tid 1 2014-07-01 17:08:04,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: writing mira0435601-125 from 4472832 to 5120000 tid 2 2014-07-01 17:08:04,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: writing mira0435601-125 from 5120000 to 5677056 tid 3 2014-07-01 17:08:04,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 64 current snap is 0 2014-07-01 17:08:04,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: seq_num 2156 ranges {4059136=585728} 2014-07-01 17:08:04,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: writing mira0435601-64 from 4059136 to 4644864 tid 1 2014-07-01 17:08:04,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3501: read oid 246 snap -1 2014-07-01 17:08:04,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3496: expect (ObjNum 1372 snap 0 seq_num 1372) 2014-07-01 17:08:04,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2740: done (2 left) 2014-07-01 17:08:04,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2741: done (1 left) 2014-07-01 17:08:04,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2742: done (0 left) 2014-07-01 17:08:04,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: snap_remove snap 244 2014-07-01 17:08:04,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: finishing write tid 1 to mira0435601-11 2014-07-01 17:08:04,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: finishing write tid 2 to mira0435601-11 2014-07-01 17:08:04,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: finishing write tid 4 to mira0435601-11 2014-07-01 17:08:04,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: finishing write tid 5 to mira0435601-11 2014-07-01 17:08:04,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 209 (ObjNum 2154 snap 0 seq_num 2154) dirty exists 2014-07-01 17:08:04,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: finishing write tid 1 to mira0435601-153 2014-07-01 17:08:04,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3501: expect (ObjNum 1733 snap 0 seq_num 1733) 2014-07-01 17:08:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2743: done (0 left) 2014-07-01 17:08:05,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: delete oid 85 current snap is 303 2014-07-01 17:08:06,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:2744: done (0 left) 2014-07-01 17:08:06,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: snap_create 2014-07-01 17:08:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2745: done (0 left) 2014-07-01 17:08:06,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:2746: read oid 201 snap -1 2014-07-01 17:08:06,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2747: read oid 420 snap 303 2014-07-01 17:08:06,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:2748: delete oid 419 current snap is 304 2014-07-01 17:08:07,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: finishing write tid 2 to mira0435601-153 2014-07-01 17:08:07,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: finishing write tid 4 to mira0435601-153 2014-07-01 17:08:07,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: finishing write tid 1 to mira0435601-125 2014-07-01 17:08:07,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: finishing write tid 2 to mira0435601-125 2014-07-01 17:08:07,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: finishing write tid 3 to mira0435601-125 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: finishing write tid 5 to mira0435601-125 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: finishing write tid 5 to mira0435601-153 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 153 v 218 (ObjNum 2153 snap 0 seq_num 2153) dirty exists 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: finishing write tid 6 to mira0435601-125 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 340 (ObjNum 2155 snap 0 seq_num 2155) dirty exists 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 1 to mira0435601-64 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 3 to mira0435601-64 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 4 to mira0435601-64 2014-07-01 17:08:07,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 240 (ObjNum 2156 snap 0 seq_num 2156) dirty exists 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3495: done (6 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3496: done (5 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: done (4 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: done (3 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: done (2 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: done (1 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3501: done (0 left) 2014-07-01 17:08:07,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: read oid 77 snap -1 2014-07-01 17:08:07,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3503: read oid 191 snap -1 2014-07-01 17:08:07,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 375 current snap is 0 2014-07-01 17:08:07,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: seq_num 2157 ranges {0=454656} 2014-07-01 17:08:07,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: writing mira0435601-375 from 0 to 454656 tid 1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: read oid 193 snap -1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3506: read oid 318 snap -1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: read oid 334 snap -1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3508: read oid 141 snap -1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 293 current snap is 0 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: seq_num 2158 ranges {0=327680} 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: writing mira0435601-293 from 0 to 327680 tid 1 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 75 current snap is 0 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: seq_num 2159 ranges {6258688=757760,7016448=471040,7487488=331776} 2014-07-01 17:08:07,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: writing mira0435601-75 from 6258688 to 7016448 tid 1 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: writing mira0435601-75 from 7016448 to 7487488 tid 2 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: writing mira0435601-75 from 7487488 to 7819264 tid 3 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 23 current snap is 0 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: seq_num 2160 ranges {692224=675840,1368064=704512,2072576=577536,2650112=323584} 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: writing mira0435601-23 from 692224 to 1368064 tid 1 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: writing mira0435601-23 from 1368064 to 2072576 tid 2 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2746: done (2 left) 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2747: done (1 left) 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2748: done (0 left) 2014-07-01 17:08:07,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:2749: delete oid 359 current snap is 304 2014-07-01 17:08:07,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: writing mira0435601-23 from 2072576 to 2650112 tid 3 2014-07-01 17:08:07,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: writing mira0435601-23 from 2650112 to 2973696 tid 4 2014-07-01 17:08:07,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:3512: read oid 339 snap -1 2014-07-01 17:08:07,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3503: expect (ObjNum 1153 snap 0 seq_num 1153) 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2749: done (0 left) 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: write oid 282 current snap is 304 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: seq_num 1323 ranges {523858=677209,1658577=542262,2708359=615499} 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0437250-282 from 523858 to 1201067 tid 1 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0437250-282 from 1658577 to 2200839 tid 2 2014-07-01 17:08:07,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: writing mira0437250-282 from 2708359 to 3323858 tid 3 2014-07-01 17:08:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: read oid 42 snap -1 2014-07-01 17:08:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: snap_create 2014-07-01 17:08:07,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: expect (ObjNum 516 snap 9 seq_num 516) 2014-07-01 17:08:07,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 1 to mira0437250-282 2014-07-01 17:08:07,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 2 to mira0437250-282 2014-07-01 17:08:07,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 3 to mira0437250-282 2014-07-01 17:08:07,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 5 to mira0437250-282 2014-07-01 17:08:07,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: finishing write tid 6 to mira0437250-282 2014-07-01 17:08:07,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 1030 (ObjNum 1323 snap 304 seq_num 1323) dirty exists 2014-07-01 17:08:07,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2750: done (2 left) 2014-07-01 17:08:07,908.908 INFO:teuthology.task.rados.rados.0.mira043.stdout:2751: done (1 left) 2014-07-01 17:08:07,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2752: done (0 left) 2014-07-01 17:08:07,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2753: delete oid 71 current snap is 305 2014-07-01 17:08:07,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2753: done (0 left) 2014-07-01 17:08:07,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: write oid 238 current snap is 305 2014-07-01 17:08:07,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: seq_num 1325 ranges {412493=400000} 2014-07-01 17:08:07,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: writing mira0437250-238 from 412493 to 812493 tid 1 2014-07-01 17:08:07,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: read oid 437 snap -1 2014-07-01 17:08:07,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: read oid 108 snap -1 2014-07-01 17:08:07,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:2757: delete oid 132 current snap is 305 2014-07-01 17:08:08,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: expect (ObjNum 107 snap 0 seq_num 107) 2014-07-01 17:08:08,941.941 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_ops_in_flight' 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:no valid command found; 10 closest matches: 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:config show 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:help 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:log dump 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:get_command_descriptions 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:git_version 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:config set [...] 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:version 2014-07-01 17:08:09,343.343 INFO:teuthology.orchestra.run.mira028.stderr:2 2014-07-01 17:08:09,344.344 INFO:teuthology.orchestra.run.mira028.stderr:config get 2014-07-01 17:08:09,344.344 INFO:teuthology.orchestra.run.mira028.stderr:0 2014-07-01 17:08:09,344.344 INFO:teuthology.orchestra.run.mira028.stderr:admin_socket: invalid command 2014-07-01 17:08:09,348.348 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 3, ['dump_ops_in_flight'] 2014-07-01 17:08:11,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: expect (ObjNum 816 snap 130 seq_num 816) 2014-07-01 17:08:12,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: expect (ObjNum 532 snap 0 seq_num 532) 2014-07-01 17:08:13,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: finishing write tid 1 to mira0437250-238 2014-07-01 17:08:13,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: finishing write tid 3 to mira0437250-238 2014-07-01 17:08:13,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: finishing write tid 4 to mira0437250-238 2014-07-01 17:08:13,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 757 (ObjNum 1325 snap 305 seq_num 1325) dirty exists 2014-07-01 17:08:13,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2754: done (3 left) 2014-07-01 17:08:13,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2755: done (2 left) 2014-07-01 17:08:13,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2756: done (1 left) 2014-07-01 17:08:13,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2757: done (0 left) 2014-07-01 17:08:13,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2758: delete oid 105 current snap is 305 2014-07-01 17:08:13,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2758: done (0 left) 2014-07-01 17:08:13,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2759: snap_create 2014-07-01 17:08:14,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2759: done (0 left) 2014-07-01 17:08:14,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:2760: delete oid 193 current snap is 306 2014-07-01 17:08:14,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2760: done (0 left) 2014-07-01 17:08:14,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:2761: snap_remove snap 268 2014-07-01 17:08:14,349.349 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_ops_in_flight' 2014-07-01 17:08:14,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: expect (ObjNum 1596 snap 0 seq_num 1596) 2014-07-01 17:08:15,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2761: done (0 left) 2014-07-01 17:08:15,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: write oid 460 current snap is 306 2014-07-01 17:08:15,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: seq_num 1329 ranges {608964=580227,1701577=507387} 2014-07-01 17:08:15,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: writing mira0437250-460 from 608964 to 1189191 tid 1 2014-07-01 17:08:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: writing mira0437250-460 from 1701577 to 2208964 tid 2 2014-07-01 17:08:15,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: snap_create 2014-07-01 17:08:15,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: finishing write tid 1 to mira0437250-460 2014-07-01 17:08:15,159.159 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: finishing write tid 2 to mira0437250-460 2014-07-01 17:08:15,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: finishing write tid 4 to mira0437250-460 2014-07-01 17:08:15,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: finishing write tid 5 to mira0437250-460 2014-07-01 17:08:15,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 1038 (ObjNum 1329 snap 306 seq_num 1329) dirty exists 2014-07-01 17:08:16,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2762: done (1 left) 2014-07-01 17:08:16,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2763: done (0 left) 2014-07-01 17:08:16,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: snap_create 2014-07-01 17:08:17,047.047 INFO:teuthology.task.rados.rados.0.mira043.stdout:2764: done (0 left) 2014-07-01 17:08:17,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: delete oid 344 current snap is 308 2014-07-01 17:08:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 1 to mira0435601-293 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 3 to mira0435601-293 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 1 to mira0435601-75 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 2 to mira0435601-75 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 3 to mira0435601-75 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 5 to mira0435601-75 2014-07-01 17:08:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: expect (ObjNum 1750 snap 0 seq_num 1750) 2014-07-01 17:08:18,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2765: done (0 left) 2014-07-01 17:08:18,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: read oid 182 snap 299 2014-07-01 17:08:18,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:2767: read oid 64 snap -1 2014-07-01 17:08:18,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2768: rollback oid 230 current snap is 308 2014-07-01 17:08:18,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 230 to 257 2014-07-01 17:08:18,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: read oid 469 snap -1 2014-07-01 17:08:18,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: read oid 154 snap -1 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2771: delete oid 56 current snap is 308 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2766: done (5 left) 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2771: done (4 left) 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2772: delete oid 112 current snap is 308 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 1415 (ObjNum 692 snap 88 seq_num 692) dirty exists 2014-07-01 17:08:18,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: expect (ObjNum 1180 snap 250 seq_num 1180) 2014-07-01 17:08:19,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: expect (ObjNum 153 snap 0 seq_num 153) 2014-07-01 17:08:19,765.765 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:08:19,765.765 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:19,765.765 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.037128 2014-07-01 17:08:19,765.765 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 0.0371278967147' 2014-07-01 17:08:21,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2767: done (4 left) 2014-07-01 17:08:21,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2768: done (3 left) 2014-07-01 17:08:21,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2769: done (2 left) 2014-07-01 17:08:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2770: done (1 left) 2014-07-01 17:08:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2772: done (0 left) 2014-07-01 17:08:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: write oid 383 current snap is 308 2014-07-01 17:08:21,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: seq_num 1333 ranges {405435=654965,1837339=416182,2682214=474292,3605434=1} 2014-07-01 17:08:21,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: writing mira0437250-383 from 405435 to 1060400 tid 1 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: writing mira0437250-383 from 1837339 to 2253521 tid 2 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: writing mira0437250-383 from 2682214 to 3156506 tid 3 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: writing mira0437250-383 from 3605434 to 3605435 tid 4 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: read oid 209 snap -1 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: write oid 368 current snap is 308 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: seq_num 1334 ranges {349240=1} 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: writing mira0437250-368 from 349240 to 349241 tid 1 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: snap_remove snap 301 2014-07-01 17:08:21,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: expect (ObjNum 208 snap 0 seq_num 208) 2014-07-01 17:08:22,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 1 to mira0437250-383 2014-07-01 17:08:22,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 2 to mira0437250-383 2014-07-01 17:08:22,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 3 to mira0437250-383 2014-07-01 17:08:22,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 4 to mira0437250-383 2014-07-01 17:08:22,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 6 to mira0437250-383 2014-07-01 17:08:22,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 1 to mira0437250-368 2014-07-01 17:08:22,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 3 to mira0437250-368 2014-07-01 17:08:22,327.327 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 0.0371279 (824332) 2014-07-01 17:08:22,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: finishing write tid 7 to mira0437250-383 2014-07-01 17:08:22,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 1420 (ObjNum 1333 snap 308 seq_num 1333) dirty exists 2014-07-01 17:08:22,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: finishing write tid 4 to mira0437250-368 2014-07-01 17:08:22,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 368 v 1195 (ObjNum 1334 snap 308 seq_num 1334) dirty exists 2014-07-01 17:08:23,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:3512: expect (ObjNum 2000 snap 0 seq_num 2000) 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2773: done (3 left) 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2774: done (2 left) 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2775: done (1 left) 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2776: done (0 left) 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: write oid 115 current snap is 308 2014-07-01 17:08:23,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: seq_num 1335 ranges {410423=599597,1444035=166388} 2014-07-01 17:08:23,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:3508: expect (ObjNum 1682 snap 0 seq_num 1682) 2014-07-01 17:08:23,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: writing mira0437250-115 from 410423 to 1010020 tid 1 2014-07-01 17:08:23,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: writing mira0437250-115 from 1444035 to 1610423 tid 2 2014-07-01 17:08:23,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: read oid 272 snap -1 2014-07-01 17:08:23,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: snap_remove snap 290 2014-07-01 17:08:23,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 1 to mira0437250-115 2014-07-01 17:08:23,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 2 to mira0437250-115 2014-07-01 17:08:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 4 to mira0435601-293 2014-07-01 17:08:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 220 (ObjNum 2158 snap 0 seq_num 2158) dirty exists 2014-07-01 17:08:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 6 to mira0435601-75 2014-07-01 17:08:23,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 280 (ObjNum 2159 snap 0 seq_num 2159) dirty exists 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 1 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: finishing write tid 1 to mira0435601-375 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 2 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 3 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: finishing write tid 3 to mira0435601-375 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 4 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 6 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: finishing write tid 4 to mira0435601-375 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 375 v 153 (ObjNum 2157 snap 0 seq_num 2157) dirty exists 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: finishing write tid 7 to mira0435601-23 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 220 (ObjNum 2160 snap 0 seq_num 2160) dirty exists 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: done (10 left) 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3503: done (9 left) 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: done (8 left) 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: done (7 left) 2014-07-01 17:08:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3506: done (6 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: done (5 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3508: done (4 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: done (3 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: done (2 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: done (1 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3512: done (0 left) 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 4 to mira0437250-115 2014-07-01 17:08:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: finishing write tid 5 to mira0437250-115 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 115 v 909 (ObjNum 1335 snap 308 seq_num 1335) dirty exists 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3513: read oid 458 snap -1 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: read oid 267 snap -1 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 455 current snap is 0 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: seq_num 2161 ranges {5672960=548864,6221824=688128,6909952=647168,7557120=270336} 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: writing mira0435601-455 from 5672960 to 6221824 tid 1 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: writing mira0435601-455 from 6221824 to 6909952 tid 2 2014-07-01 17:08:23,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: writing mira0435601-455 from 6909952 to 7557120 tid 3 2014-07-01 17:08:23,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: writing mira0435601-455 from 7557120 to 7827456 tid 4 2014-07-01 17:08:23,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 350 current snap is 0 2014-07-01 17:08:23,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: seq_num 2162 ranges {4698112=577536,5275648=401408} 2014-07-01 17:08:23,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: writing mira0435601-350 from 4698112 to 5275648 tid 1 2014-07-01 17:08:23,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: writing mira0435601-350 from 5275648 to 5677056 tid 2 2014-07-01 17:08:23,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 163 current snap is 0 2014-07-01 17:08:23,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: seq_num 2163 ranges {3624960=446464,4071424=401408} 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: writing mira0435601-163 from 3624960 to 4071424 tid 1 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: writing mira0435601-163 from 4071424 to 4472832 tid 2 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 143 current snap is 0 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: seq_num 2164 ranges {1474560=770048,2244608=684032,2928640=118784} 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: writing mira0435601-143 from 1474560 to 2244608 tid 1 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: writing mira0435601-143 from 2244608 to 2928640 tid 2 2014-07-01 17:08:23,663.663 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: writing mira0435601-143 from 2928640 to 3047424 tid 3 2014-07-01 17:08:23,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 25 current snap is 0 2014-07-01 17:08:23,664.664 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: seq_num 2165 ranges {1658880=393216} 2014-07-01 17:08:23,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: writing mira0435601-25 from 1658880 to 2052096 tid 1 2014-07-01 17:08:23,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3520: delete oid 423 current snap is 0 2014-07-01 17:08:26,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2777: done (2 left) 2014-07-01 17:08:26,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2778: done (1 left) 2014-07-01 17:08:26,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2779: done (0 left) 2014-07-01 17:08:26,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: delete oid 245 current snap is 308 2014-07-01 17:08:26,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 1 to mira0435601-350 2014-07-01 17:08:26,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 2 to mira0435601-350 2014-07-01 17:08:26,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: expect (ObjNum 2143 snap 0 seq_num 2143) 2014-07-01 17:08:26,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2780: done (0 left) 2014-07-01 17:08:26,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:2781: snap_create 2014-07-01 17:08:27,359.359 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:08:27,359.359 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:27,359.359 INFO:teuthology.task.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2014-07-01 17:08:27,359.359 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 3 0' 2014-07-01 17:08:27,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2781: done (0 left) 2014-07-01 17:08:27,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: write oid 497 current snap is 309 2014-07-01 17:08:27,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: seq_num 1337 ranges {450482=579302,1672299=378183} 2014-07-01 17:08:27,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: writing mira0437250-497 from 450482 to 1029784 tid 1 2014-07-01 17:08:27,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: writing mira0437250-497 from 1672299 to 2050482 tid 2 2014-07-01 17:08:27,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: write oid 183 current snap is 309 2014-07-01 17:08:27,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: seq_num 1338 ranges {794288=552130,1883583=670006,2794287=1} 2014-07-01 17:08:27,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: writing mira0437250-183 from 794288 to 1346418 tid 1 2014-07-01 17:08:27,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: writing mira0437250-183 from 1883583 to 2553589 tid 2 2014-07-01 17:08:27,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: writing mira0437250-183 from 2794287 to 2794288 tid 3 2014-07-01 17:08:27,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: delete oid 190 current snap is 309 2014-07-01 17:08:28,450.450 INFO:teuthology.orchestra.run.mira079.stderr:set osd.3 primary-affinity to 0 (802) 2014-07-01 17:08:29,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 4 to mira0435601-350 2014-07-01 17:08:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 1 to mira0435601-455 2014-07-01 17:08:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: finishing write tid 1 to mira0435601-143 2014-07-01 17:08:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 2 to mira0435601-455 2014-07-01 17:08:29,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 5 to mira0435601-350 2014-07-01 17:08:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 350 v 291 (ObjNum 2162 snap 0 seq_num 2162) dirty exists 2014-07-01 17:08:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 3 to mira0435601-455 2014-07-01 17:08:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: finishing write tid 2 to mira0435601-143 2014-07-01 17:08:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: finishing write tid 3 to mira0435601-143 2014-07-01 17:08:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: done (7 left) 2014-07-01 17:08:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: done (6 left) 2014-07-01 17:08:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3520: done (5 left) 2014-07-01 17:08:29,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3521: read oid 217 snap -1 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 4 to mira0435601-455 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3522: read oid 231 snap -1 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: finishing write tid 5 to mira0435601-143 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: finishing write tid 1 to mira0435601-25 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 6 to mira0435601-455 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: finishing write tid 3 to mira0435601-25 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: finishing write tid 6 to mira0435601-143 2014-07-01 17:08:29,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 166 (ObjNum 2164 snap 0 seq_num 2164) dirty exists 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: finishing write tid 7 to mira0435601-455 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 455 v 345 (ObjNum 2161 snap 0 seq_num 2161) dirty exists 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: finishing write tid 4 to mira0435601-25 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 321 (ObjNum 2165 snap 0 seq_num 2165) dirty exists 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: done (6 left) 2014-07-01 17:08:30,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: done (5 left) 2014-07-01 17:08:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: done (4 left) 2014-07-01 17:08:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 17:08:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: seq_num 2167 ranges {1503232=139264} 2014-07-01 17:08:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: writing mira0435601-323 from 1503232 to 1642496 tid 1 2014-07-01 17:08:30,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: delete oid 186 current snap is 0 2014-07-01 17:08:30,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3522: expect (ObjNum 1608 snap 0 seq_num 1608) 2014-07-01 17:08:30,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 1 to mira0437250-183 2014-07-01 17:08:30,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 2 to mira0437250-183 2014-07-01 17:08:31,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2784: done (2 left) 2014-07-01 17:08:31,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: write oid 185 current snap is 309 2014-07-01 17:08:31,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: seq_num 1340 ranges {550699=647163,1942503=410397,2917799=519585,3750698=1} 2014-07-01 17:08:31,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: writing mira0437250-185 from 550699 to 1197862 tid 1 2014-07-01 17:08:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: writing mira0437250-185 from 1942503 to 2352900 tid 2 2014-07-01 17:08:31,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: writing mira0437250-185 from 2917799 to 3437384 tid 3 2014-07-01 17:08:31,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: writing mira0437250-185 from 3750698 to 3750699 tid 4 2014-07-01 17:08:31,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: rollback oid 404 current snap is 309 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 404 to 304 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: rollback oid 429 current snap is 309 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 429 to 285 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: read oid 102 snap -1 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: read oid 312 snap -1 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2790: delete oid 205 current snap is 309 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 1 to mira0437250-497 2014-07-01 17:08:31,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: expect (ObjNum 731 snap 99 seq_num 731) 2014-07-01 17:08:31,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:3521: expect (ObjNum 1646 snap 0 seq_num 1646) 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 2 to mira0437250-497 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 4 to mira0437250-497 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: finishing write tid 5 to mira0437250-497 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 497 v 1244 (ObjNum 1337 snap 309 seq_num 1337) dirty exists 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 1 to mira0437250-185 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 579 (ObjNum 960 snap 175 seq_num 960) dirty exists 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 1137 (ObjNum 1187 snap 251 seq_num 1187) dirty exists 2014-07-01 17:08:31,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 2 to mira0437250-185 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2782: done (7 left) 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2786: done (6 left) 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2787: done (5 left) 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2788: done (4 left) 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2790: done (3 left) 2014-07-01 17:08:31,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: write oid 74 current snap is 309 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: seq_num 1342 ranges {190758=1} 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: writing mira0437250-74 from 190758 to 190759 tid 1 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2792: delete oid 353 current snap is 309 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 3 to mira0437250-185 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 4 to mira0437250-185 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 6 to mira0437250-185 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: finishing write tid 7 to mira0437250-185 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 185 v 1450 (ObjNum 1340 snap 309 seq_num 1340) dirty exists 2014-07-01 17:08:31,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: expect (ObjNum 1134 snap 232 seq_num 1134) 2014-07-01 17:08:32,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 3 to mira0437250-183 2014-07-01 17:08:32,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 5 to mira0437250-183 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: finishing write tid 6 to mira0437250-183 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 863 (ObjNum 1338 snap 309 seq_num 1338) dirty exists 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2783: done (4 left) 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2785: done (3 left) 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2789: done (2 left) 2014-07-01 17:08:32,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2792: done (1 left) 2014-07-01 17:08:32,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: read oid 266 snap -1 2014-07-01 17:08:32,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: write oid 66 current snap is 309 2014-07-01 17:08:32,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: seq_num 1344 ranges {747170=400000} 2014-07-01 17:08:32,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: writing mira0437250-66 from 747170 to 1147170 tid 1 2014-07-01 17:08:32,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: write oid 296 current snap is 309 2014-07-01 17:08:32,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: expect (ObjNum 265 snap 0 seq_num 265) 2014-07-01 17:08:32,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: finishing write tid 1 to mira0437250-74 2014-07-01 17:08:32,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: finishing write tid 3 to mira0437250-74 2014-07-01 17:08:32,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: finishing write tid 1 to mira0437250-66 2014-07-01 17:08:32,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: finishing write tid 3 to mira0437250-66 2014-07-01 17:08:32,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: seq_num 1345 ranges {692000=729801,1891999=1} 2014-07-01 17:08:32,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: writing mira0437250-296 from 692000 to 1421801 tid 1 2014-07-01 17:08:32,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: writing mira0437250-296 from 1891999 to 1892000 tid 2 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2793: done (3 left) 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2796: rollback oid 213 current snap is 309 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 213 to 278 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: finishing write tid 4 to mira0437250-74 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 1718 (ObjNum 1342 snap 309 seq_num 1342) dirty exists 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2791: done (3 left) 2014-07-01 17:08:32,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: write oid 195 current snap is 309 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: seq_num 1346 ranges {634782=550181,1734880=618064,2634781=1} 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: writing mira0437250-195 from 634782 to 1184963 tid 1 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: writing mira0437250-195 from 1734880 to 2352944 tid 2 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: writing mira0437250-195 from 2634781 to 2634782 tid 3 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: write oid 301 current snap is 309 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: seq_num 1347 ranges {731036=673409,1965364=365672} 2014-07-01 17:08:32,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: writing mira0437250-301 from 731036 to 1404445 tid 1 2014-07-01 17:08:32,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: writing mira0437250-301 from 1965364 to 2331036 tid 2 2014-07-01 17:08:33,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: write oid 340 current snap is 309 2014-07-01 17:08:33,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: seq_num 1348 ranges {792217=645214,2044424=509878,3005554=586663} 2014-07-01 17:08:33,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0437250-340 from 792217 to 1437431 tid 1 2014-07-01 17:08:33,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0437250-340 from 2044424 to 2554302 tid 2 2014-07-01 17:08:33,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: writing mira0437250-340 from 3005554 to 3592217 tid 3 2014-07-01 17:08:33,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: read oid 197 snap -1 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 1437 (ObjNum 1177 snap 247 seq_num 1177) dirty exists 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 1 to mira0437250-340 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: finishing write tid 4 to mira0437250-66 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 66 v 1070 (ObjNum 1344 snap 309 seq_num 1344) dirty exists 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 2 to mira0437250-340 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2794: done (6 left) 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2796: done (5 left) 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: write oid 125 current snap is 309 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: seq_num 1349 ranges {336022=1} 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: writing mira0437250-125 from 336022 to 336023 tid 1 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2802: rollback oid 130 current snap is 309 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 130 to 279 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: write oid 1 current snap is 309 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: seq_num 1350 ranges {32276=1} 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: writing mira0437250-1 from 32276 to 32277 tid 1 2014-07-01 17:08:33,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: delete oid 252 current snap is 309 2014-07-01 17:08:33,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 3 to mira0437250-340 2014-07-01 17:08:33,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 5 to mira0437250-340 2014-07-01 17:08:33,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: finishing write tid 6 to mira0437250-340 2014-07-01 17:08:33,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 915 (ObjNum 1348 snap 309 seq_num 1348) dirty exists 2014-07-01 17:08:33,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 130 v 775 (ObjNum 994 snap 189 seq_num 994) dirty exists 2014-07-01 17:08:33,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: finishing write tid 1 to mira0437250-125 2014-07-01 17:08:33,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: finishing write tid 3 to mira0437250-125 2014-07-01 17:08:33,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 1 to mira0437250-1 2014-07-01 17:08:33,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 3 to mira0437250-1 2014-07-01 17:08:33,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2799: done (8 left) 2014-07-01 17:08:33,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:2800: done (7 left) 2014-07-01 17:08:33,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2802: done (6 left) 2014-07-01 17:08:33,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2804: done (5 left) 2014-07-01 17:08:33,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: write oid 76 current snap is 309 2014-07-01 17:08:33,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: seq_num 1352 ranges {720912=720975,1520911=1} 2014-07-01 17:08:33,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: writing mira0437250-76 from 720912 to 1441887 tid 1 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: writing mira0437250-76 from 1520911 to 1520912 tid 2 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2806: snap_remove snap 258 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: finishing write tid 4 to mira0437250-125 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 699 (ObjNum 1349 snap 309 seq_num 1349) dirty exists 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: finishing write tid 1 to mira0437250-296 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: finishing write tid 2 to mira0437250-296 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: finishing write tid 4 to mira0437250-296 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: finishing write tid 1 to mira0437250-76 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: finishing write tid 2 to mira0437250-76 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: finishing write tid 4 to mira0437250-76 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: finishing write tid 4 to mira0437250-1 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 760 (ObjNum 1350 snap 309 seq_num 1350) dirty exists 2014-07-01 17:08:33,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: finishing write tid 5 to mira0437250-76 2014-07-01 17:08:33,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 1654 (ObjNum 1352 snap 309 seq_num 1352) dirty exists 2014-07-01 17:08:33,465.465 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:08:33,465.465 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:33,465.465 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_2 2014-07-01 17:08:33,465.465 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:08:33,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2801: done (6 left) 2014-07-01 17:08:33,749.749 INFO:teuthology.task.rados.rados.0.mira043.stdout:2803: done (5 left) 2014-07-01 17:08:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:2805: done (4 left) 2014-07-01 17:08:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:2806: done (3 left) 2014-07-01 17:08:33,750.750 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: snap_remove snap 306 2014-07-01 17:08:34,191.191 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:08:34,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:2807: done (3 left) 2014-07-01 17:08:34,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: rollback oid 373 current snap is 309 2014-07-01 17:08:34,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 373 to 272 2014-07-01 17:08:34,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: snap_remove snap 259 2014-07-01 17:08:35,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 1 to mira0435601-323 2014-07-01 17:08:35,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 3 to mira0435601-323 2014-07-01 17:08:35,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 4 to mira0435601-323 2014-07-01 17:08:35,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 210 (ObjNum 2167 snap 0 seq_num 2167) dirty exists 2014-07-01 17:08:35,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3513: expect (ObjNum 1232 snap 0 seq_num 1232) 2014-07-01 17:08:37,603.603 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pgp_num 66' 2014-07-01 17:08:38,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: finishing write tid 1 to mira0435601-163 2014-07-01 17:08:38,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: finishing write tid 2 to mira0435601-163 2014-07-01 17:08:38,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: finishing write tid 4 to mira0435601-163 2014-07-01 17:08:38,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: finishing write tid 5 to mira0435601-163 2014-07-01 17:08:38,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 386 (ObjNum 2163 snap 0 seq_num 2163) dirty exists 2014-07-01 17:08:38,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3513: done (5 left) 2014-07-01 17:08:38,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: done (4 left) 2014-07-01 17:08:38,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3521: done (3 left) 2014-07-01 17:08:38,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3522: done (2 left) 2014-07-01 17:08:38,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: done (1 left) 2014-07-01 17:08:38,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: done (0 left) 2014-07-01 17:08:38,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 80 current snap is 0 2014-07-01 17:08:38,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: seq_num 2169 ranges {3354624=487424,3842048=491520,4333568=540672,4874240=499712,5373952=73728} 2014-07-01 17:08:38,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: writing mira0435601-80 from 3354624 to 3842048 tid 1 2014-07-01 17:08:38,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: writing mira0435601-80 from 3842048 to 4333568 tid 2 2014-07-01 17:08:38,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: writing mira0435601-80 from 4333568 to 4874240 tid 3 2014-07-01 17:08:38,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: writing mira0435601-80 from 4874240 to 5373952 tid 4 2014-07-01 17:08:38,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: writing mira0435601-80 from 5373952 to 5447680 tid 5 2014-07-01 17:08:38,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3526: read oid 281 snap -1 2014-07-01 17:08:38,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3527: delete oid 382 current snap is 0 2014-07-01 17:08:40,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:2809: done (4 left) 2014-07-01 17:08:40,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: read oid 391 snap -1 2014-07-01 17:08:40,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: delete oid 59 current snap is 309 2014-07-01 17:08:40,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 1440 (ObjNum 372 snap 0 seq_num 372) dirty exists 2014-07-01 17:08:40,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2808: done (5 left) 2014-07-01 17:08:40,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2811: done (4 left) 2014-07-01 17:08:40,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: write oid 328 current snap is 309 2014-07-01 17:08:40,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: seq_num 1354 ranges {476300=699656,1589250=716521,2476299=1} 2014-07-01 17:08:40,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: writing mira0437250-328 from 476300 to 1175956 tid 1 2014-07-01 17:08:40,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: writing mira0437250-328 from 1589250 to 2305771 tid 2 2014-07-01 17:08:40,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: writing mira0437250-328 from 2476299 to 2476300 tid 3 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: read oid 382 snap -1 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: finishing write tid 1 to mira0437250-328 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: rollback oid 279 current snap is 309 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 279 to 285 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2815: snap_remove snap 280 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: finishing write tid 2 to mira0437250-328 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: finishing write tid 3 to mira0437250-328 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: finishing write tid 5 to mira0437250-328 2014-07-01 17:08:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: expect (ObjNum 1057 snap 211 seq_num 1057) 2014-07-01 17:08:41,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: finishing write tid 6 to mira0437250-328 2014-07-01 17:08:41,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 1398 (ObjNum 1354 snap 309 seq_num 1354) dirty exists 2014-07-01 17:08:41,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:2812: done (7 left) 2014-07-01 17:08:41,331.331 INFO:teuthology.task.rados.rados.0.mira043.stdout:2813: done (6 left) 2014-07-01 17:08:41,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:2815: done (5 left) 2014-07-01 17:08:41,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: delete oid 184 current snap is 309 2014-07-01 17:08:41,338.338 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pgp_num to 66 2014-07-01 17:08:41,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: finishing write tid 1 to mira0437250-195 2014-07-01 17:08:41,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: finishing write tid 2 to mira0437250-195 2014-07-01 17:08:41,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: finishing write tid 3 to mira0437250-195 2014-07-01 17:08:41,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:2816: done (5 left) 2014-07-01 17:08:41,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: snap_remove snap 308 2014-07-01 17:08:41,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: finishing write tid 5 to mira0437250-195 2014-07-01 17:08:41,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: finishing write tid 6 to mira0437250-195 2014-07-01 17:08:41,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 727 (ObjNum 1346 snap 309 seq_num 1346) dirty exists 2014-07-01 17:08:41,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: finishing write tid 5 to mira0437250-296 2014-07-01 17:08:41,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 1047 (ObjNum 1345 snap 309 seq_num 1345) dirty exists 2014-07-01 17:08:41,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: expect (ObjNum 907 snap 158 seq_num 907) 2014-07-01 17:08:41,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 279 v 0 (ObjNum 3 snap 0 seq_num 0) dirty dne 2014-07-01 17:08:41,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:3526: expect (ObjNum 1855 snap 0 seq_num 1855) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2795: done (5 left) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2797: done (4 left) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2810: done (3 left) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2814: done (2 left) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2817: done (1 left) 2014-07-01 17:08:42,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2818: delete oid 431 current snap is 309 2014-07-01 17:08:42,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:2818: done (1 left) 2014-07-01 17:08:42,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:2819: read oid 467 snap -1 2014-07-01 17:08:42,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:2820: rollback oid 74 current snap is 309 2014-07-01 17:08:42,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 74 to 291 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: read oid 473 snap -1 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: read oid 396 snap -1 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: write oid 333 current snap is 309 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: seq_num 1357 ranges {177540=1} 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: writing mira0437250-333 from 177540 to 177541 tid 1 2014-07-01 17:08:42,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: delete oid 202 current snap is 309 2014-07-01 17:08:42,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: finishing write tid 1 to mira0437250-301 2014-07-01 17:08:42,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: finishing write tid 2 to mira0437250-301 2014-07-01 17:08:42,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: finishing write tid 4 to mira0437250-301 2014-07-01 17:08:42,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: finishing write tid 5 to mira0437250-301 2014-07-01 17:08:42,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 1035 (ObjNum 1347 snap 309 seq_num 1347) dirty exists 2014-07-01 17:08:42,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 1758 (ObjNum 1280 snap 284 seq_num 1280) dirty exists 2014-07-01 17:08:42,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3526: done (2 left) 2014-07-01 17:08:42,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3527: done (1 left) 2014-07-01 17:08:42,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3528: read oid 28 snap -1 2014-07-01 17:08:42,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: read oid 297 snap -1 2014-07-01 17:08:42,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 171 current snap is 0 2014-07-01 17:08:42,890.890 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: seq_num 2171 ranges {3735552=786432} 2014-07-01 17:08:43,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: writing mira0435601-171 from 3735552 to 4521984 tid 1 2014-07-01 17:08:43,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3531: delete oid 8 current snap is 0 2014-07-01 17:08:46,366.366 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:08:46,367.367 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:46,367.367 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 17:08:46,367.367 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:08:46,367.367 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:08:46,368.368 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 17:08:48,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: finishing write tid 1 to mira0435601-171 2014-07-01 17:08:48,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: finishing write tid 3 to mira0435601-171 2014-07-01 17:08:48,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3528: done (4 left) 2014-07-01 17:08:48,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: done (3 left) 2014-07-01 17:08:48,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3531: done (2 left) 2014-07-01 17:08:48,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 17:08:48,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: seq_num 2173 ranges {6492160=335872} 2014-07-01 17:08:48,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: writing mira0435601-58 from 6492160 to 6828032 tid 1 2014-07-01 17:08:48,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 76 current snap is 0 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: seq_num 2174 ranges {4120576=204800} 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: writing mira0435601-76 from 4120576 to 4325376 tid 1 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 93 current snap is 0 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: seq_num 2175 ranges {4370432=77824} 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: writing mira0435601-93 from 4370432 to 4448256 tid 1 2014-07-01 17:08:48,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3535: delete oid 473 current snap is 0 2014-07-01 17:08:48,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: finishing write tid 1 to mira0435601-76 2014-07-01 17:08:48,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: finishing write tid 3 to mira0435601-76 2014-07-01 17:08:48,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3535: done (5 left) 2014-07-01 17:08:48,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: read oid 353 snap -1 2014-07-01 17:08:48,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 41 current snap is 0 2014-07-01 17:08:48,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: seq_num 2177 ranges {1417216=778240,2195456=401408} 2014-07-01 17:08:48,933.933 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: writing mira0435601-41 from 1417216 to 2195456 tid 1 2014-07-01 17:08:48,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2798: done (6 left) 2014-07-01 17:08:48,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2820: done (5 left) 2014-07-01 17:08:48,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2821: done (4 left) 2014-07-01 17:08:48,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2824: done (3 left) 2014-07-01 17:08:48,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: write oid 227 current snap is 309 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: seq_num 1359 ranges {617599=1} 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: writing mira0437250-227 from 617599 to 617600 tid 1 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: write oid 277 current snap is 309 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: seq_num 1360 ranges {562430=719026,1362429=1} 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: writing mira0437250-277 from 562430 to 1281456 tid 1 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: writing mira0437250-277 from 1362429 to 1362430 tid 2 2014-07-01 17:08:49,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: delete oid 139 current snap is 309 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2827: done (5 left) 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2828: snap_create 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2828: done (5 left) 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:2829: read oid 206 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: writing mira0435601-41 from 2195456 to 2596864 tid 2 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: read oid 86 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: finishing write tid 1 to mira0435601-93 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: finishing write tid 3 to mira0435601-93 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 1 to mira0435601-80 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 2 to mira0435601-80 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 3 to mira0435601-80 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3539: read oid 4 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: read oid 415 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: read oid 461 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: read oid 199 snap -1 2014-07-01 17:08:49,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 151 current snap is 0 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: seq_num 2178 ranges {3244032=700416,3944448=528384,4472832=675840} 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: writing mira0435601-151 from 3244032 to 3944448 tid 1 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: writing mira0435601-151 from 3944448 to 4472832 tid 2 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: writing mira0435601-151 from 4472832 to 5148672 tid 3 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3544: read oid 120 snap -1 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 4 to mira0435601-80 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 5 to mira0435601-80 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 7 to mira0435601-80 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 213 current snap is 0 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: seq_num 2179 ranges {1777664=729088} 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: writing mira0435601-213 from 1777664 to 2506752 tid 1 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 398 current snap is 0 2014-07-01 17:08:49,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: seq_num 2180 ranges {2469888=598016} 2014-07-01 17:08:49,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: read oid 61 snap 278 2014-07-01 17:08:49,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: snap_create 2014-07-01 17:08:49,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: writing mira0435601-398 from 2469888 to 3067904 tid 1 2014-07-01 17:08:49,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 17:08:49,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: expect (ObjNum 2104 snap 0 seq_num 2104) 2014-07-01 17:08:49,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: expect (ObjNum 1126 snap 229 seq_num 1126) 2014-07-01 17:08:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2829: done (7 left) 2014-07-01 17:08:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2830: done (6 left) 2014-07-01 17:08:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2831: done (5 left) 2014-07-01 17:08:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: write oid 422 current snap is 311 2014-07-01 17:08:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: seq_num 1362 ranges {718842=697707,2057123=261719} 2014-07-01 17:08:50,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: writing mira0437250-422 from 718842 to 1416549 tid 1 2014-07-01 17:08:50,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: writing mira0437250-422 from 2057123 to 2318842 tid 2 2014-07-01 17:08:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:2833: read oid 59 snap -1 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: delete oid 13 current snap is 311 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: finishing write tid 1 to mira0437250-422 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: finishing write tid 2 to mira0437250-422 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: finishing write tid 4 to mira0437250-422 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: finishing write tid 5 to mira0437250-422 2014-07-01 17:08:50,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 1020 (ObjNum 1362 snap 311 seq_num 1362) dirty exists 2014-07-01 17:08:51,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 1 to mira0437250-277 2014-07-01 17:08:51,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 2 to mira0437250-277 2014-07-01 17:08:51,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 4 to mira0437250-277 2014-07-01 17:08:51,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 1 to mira0437250-227 2014-07-01 17:08:51,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 3 to mira0437250-227 2014-07-01 17:08:51,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: finishing write tid 4 to mira0437250-227 2014-07-01 17:08:51,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 1247 (ObjNum 1359 snap 309 seq_num 1359) dirty exists 2014-07-01 17:08:51,540.540 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: expect (ObjNum 1069 snap 0 seq_num 1069) 2014-07-01 17:08:51,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: finishing write tid 5 to mira0437250-277 2014-07-01 17:08:51,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 883 (ObjNum 1360 snap 309 seq_num 1360) dirty exists 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2825: done (7 left) 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2826: done (6 left) 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2832: done (5 left) 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2833: done (4 left) 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2834: done (3 left) 2014-07-01 17:08:51,702.702 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: delete oid 61 current snap is 311 2014-07-01 17:08:51,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2819: expect (ObjNum 1242 snap 272 seq_num 1242) 2014-07-01 17:08:51,836.836 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:08:51,836.836 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:08:51,836.836 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 17:08:51,837.837 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:08:52,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: finishing write tid 1 to mira0437250-333 2014-07-01 17:08:52,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: finishing write tid 3 to mira0437250-333 2014-07-01 17:08:52,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: finishing write tid 4 to mira0437250-333 2014-07-01 17:08:52,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 855 (ObjNum 1357 snap 309 seq_num 1357) dirty exists 2014-07-01 17:08:52,602.602 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:08:53,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: finishing write tid 8 to mira0435601-80 2014-07-01 17:08:53,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 392 (ObjNum 2169 snap 0 seq_num 2169) dirty exists 2014-07-01 17:08:53,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: expect (ObjNum 1767 snap 0 seq_num 1767) 2014-07-01 17:08:56,033.033 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:08:56,033.033 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pg_num 64' 2014-07-01 17:08:56,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: finishing write tid 1 to mira0435601-213 2014-07-01 17:08:56,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: finishing write tid 3 to mira0435601-213 2014-07-01 17:08:56,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: finishing write tid 4 to mira0435601-213 2014-07-01 17:08:56,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 211 (ObjNum 2179 snap 0 seq_num 2179) dirty exists 2014-07-01 17:08:56,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: expect (ObjNum 1591 snap 0 seq_num 1591) 2014-07-01 17:08:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: finishing write tid 1 to mira0435601-41 2014-07-01 17:08:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: finishing write tid 2 to mira0435601-41 2014-07-01 17:08:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: finishing write tid 4 to mira0435601-76 2014-07-01 17:08:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 76 v 291 (ObjNum 2174 snap 0 seq_num 2174) dirty exists 2014-07-01 17:08:56,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: finishing write tid 4 to mira0435601-171 2014-07-01 17:08:56,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 339 (ObjNum 2171 snap 0 seq_num 2171) dirty exists 2014-07-01 17:08:56,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: expect (ObjNum 414 snap 0 seq_num 414) 2014-07-01 17:08:58,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3539: expect (ObjNum 2036 snap 0 seq_num 2036) 2014-07-01 17:09:00,929.929 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pg_num to 64 2014-07-01 17:09:00,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: expect (ObjNum 519 snap 10 seq_num 519) 2014-07-01 17:09:00,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: finishing write tid 4 to mira0435601-93 2014-07-01 17:09:00,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 93 v 136 (ObjNum 2175 snap 0 seq_num 2175) dirty exists 2014-07-01 17:09:00,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3544: expect (ObjNum 1629 snap 0 seq_num 1629) 2014-07-01 17:09:03,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:2819: done (3 left) 2014-07-01 17:09:03,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:2822: done (2 left) 2014-07-01 17:09:03,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:2823: done (1 left) 2014-07-01 17:09:03,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2835: done (0 left) 2014-07-01 17:09:03,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: read oid 7 snap -1 2014-07-01 17:09:03,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: write oid 427 current snap is 311 2014-07-01 17:09:03,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: seq_num 1365 ranges {702707=400640,1847071=778012,3382290=120417} 2014-07-01 17:09:03,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: writing mira0437250-427 from 702707 to 1103347 tid 1 2014-07-01 17:09:03,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: writing mira0437250-427 from 1847071 to 2625083 tid 2 2014-07-01 17:09:03,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: writing mira0437250-427 from 3382290 to 3502707 tid 3 2014-07-01 17:09:03,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: write oid 213 current snap is 311 2014-07-01 17:09:03,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: seq_num 1366 ranges {763888=1} 2014-07-01 17:09:03,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: writing mira0437250-213 from 763888 to 763889 tid 1 2014-07-01 17:09:03,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: read oid 240 snap -1 2014-07-01 17:09:03,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: expect (ObjNum 1124 snap 228 seq_num 1124) 2014-07-01 17:09:03,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: finishing write tid 1 to mira0437250-427 2014-07-01 17:09:03,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: finishing write tid 2 to mira0437250-427 2014-07-01 17:09:03,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: finishing write tid 3 to mira0437250-427 2014-07-01 17:09:03,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: finishing write tid 5 to mira0437250-427 2014-07-01 17:09:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2836: done (3 left) 2014-07-01 17:09:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2839: done (2 left) 2014-07-01 17:09:03,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2840: snap_remove snap 246 2014-07-01 17:09:03,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2840: done (2 left) 2014-07-01 17:09:03,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: write oid 403 current snap is 311 2014-07-01 17:09:03,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: seq_num 1367 ranges {459117=1} 2014-07-01 17:09:03,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: writing mira0437250-403 from 459117 to 459118 tid 1 2014-07-01 17:09:03,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:2842: delete oid 232 current snap is 311 2014-07-01 17:09:03,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2842: done (3 left) 2014-07-01 17:09:03,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: write oid 398 current snap is 311 2014-07-01 17:09:03,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: seq_num 1369 ranges {747754=724977,1947753=1} 2014-07-01 17:09:04,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0437250-398 from 747754 to 1472731 tid 1 2014-07-01 17:09:04,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: writing mira0437250-398 from 1947753 to 1947754 tid 2 2014-07-01 17:09:04,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: read oid 124 snap -1 2014-07-01 17:09:04,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2845: snap_remove snap 257 2014-07-01 17:09:04,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: expect (ObjNum 977 snap 181 seq_num 977) 2014-07-01 17:09:04,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2844: done (5 left) 2014-07-01 17:09:04,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2845: done (4 left) 2014-07-01 17:09:04,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: read oid 122 snap -1 2014-07-01 17:09:04,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:2847: snap_remove snap 276 2014-07-01 17:09:05,943.943 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:09:05,944.944 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:09:05,944.944 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.589216 2014-07-01 17:09:05,944.944 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0.589216088675' 2014-07-01 17:09:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: finishing write tid 4 to mira0435601-41 2014-07-01 17:09:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: finishing write tid 1 to mira0435601-398 2014-07-01 17:09:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: finishing write tid 5 to mira0435601-41 2014-07-01 17:09:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 283 (ObjNum 2177 snap 0 seq_num 2177) dirty exists 2014-07-01 17:09:06,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: finishing write tid 1 to mira0435601-151 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: finishing write tid 2 to mira0435601-151 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: finishing write tid 3 to mira0435601-151 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: finishing write tid 5 to mira0435601-151 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: finishing write tid 3 to mira0435601-398 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: finishing write tid 4 to mira0435601-398 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 108 (ObjNum 2180 snap 0 seq_num 2180) dirty exists 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: finishing write tid 6 to mira0435601-151 2014-07-01 17:09:06,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 287 (ObjNum 2178 snap 0 seq_num 2178) dirty exists 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: finishing write tid 1 to mira0435601-58 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: finishing write tid 3 to mira0435601-58 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: finishing write tid 4 to mira0435601-58 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 162 (ObjNum 2173 snap 0 seq_num 2173) dirty exists 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: done (15 left) 2014-07-01 17:09:06,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: done (14 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: done (13 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: done (12 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: done (11 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: done (10 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: done (9 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: done (8 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3539: done (7 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: done (6 left) 2014-07-01 17:09:06,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: done (5 left) 2014-07-01 17:09:06,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: done (4 left) 2014-07-01 17:09:06,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: done (3 left) 2014-07-01 17:09:06,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3544: done (2 left) 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: done (1 left) 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: done (0 left) 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 461 current snap is 0 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: seq_num 2181 ranges {3641344=471040} 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: writing mira0435601-461 from 3641344 to 4112384 tid 1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3548: read oid 325 snap -1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3549: read oid 176 snap -1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3550: read oid 138 snap -1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: read oid 81 snap -1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: read oid 454 snap -1 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 116 current snap is 0 2014-07-01 17:09:06,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: seq_num 2182 ranges {1855488=147456} 2014-07-01 17:09:06,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: writing mira0435601-116 from 1855488 to 2002944 tid 1 2014-07-01 17:09:06,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3554: delete oid 423 current snap is 0 2014-07-01 17:09:06,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: finishing write tid 6 to mira0437250-427 2014-07-01 17:09:06,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 1501 (ObjNum 1365 snap 311 seq_num 1365) dirty exists 2014-07-01 17:09:06,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 1 to mira0437250-403 2014-07-01 17:09:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2837: done (5 left) 2014-07-01 17:09:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2847: done (4 left) 2014-07-01 17:09:06,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2848: snap_remove snap 295 2014-07-01 17:09:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 1 to mira0437250-398 2014-07-01 17:09:06,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 2 to mira0437250-398 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 4 to mira0437250-398 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 3 to mira0437250-403 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: expect (ObjNum 1055 snap 210 seq_num 1055) 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3548: done (7 left) 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3550: done (6 left) 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3554: done (5 left) 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 277 current snap is 0 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: seq_num 2184 ranges {3563520=692224,4255744=675840,4931584=655360,5586944=274432} 2014-07-01 17:09:06,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0435601-277 from 3563520 to 4255744 tid 1 2014-07-01 17:09:06,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0435601-277 from 4255744 to 4931584 tid 2 2014-07-01 17:09:06,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0435601-277 from 4931584 to 5586944 tid 3 2014-07-01 17:09:06,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0435601-277 from 5586944 to 5861376 tid 4 2014-07-01 17:09:06,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 247 current snap is 0 2014-07-01 17:09:06,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: seq_num 2185 ranges {0=720896,720896=401408} 2014-07-01 17:09:06,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: writing mira0435601-247 from 0 to 720896 tid 1 2014-07-01 17:09:06,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: writing mira0435601-247 from 720896 to 1122304 tid 2 2014-07-01 17:09:06,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: read oid 14 snap -1 2014-07-01 17:09:06,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3558: read oid 200 snap -1 2014-07-01 17:09:06,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 393 current snap is 0 2014-07-01 17:09:06,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: seq_num 2186 ranges {0=745472,745472=491520,1236992=602112,1839104=491520,2330624=20480} 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: writing mira0435601-393 from 0 to 745472 tid 1 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: writing mira0435601-393 from 745472 to 1236992 tid 2 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: writing mira0435601-393 from 1236992 to 1839104 tid 3 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: writing mira0435601-393 from 1839104 to 2330624 tid 4 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: writing mira0435601-393 from 2330624 to 2351104 tid 5 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: seq_num 2187 ranges {6094848=512000,6606848=425984,7032832=425984,7458816=352256} 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: writing mira0435601-31 from 6094848 to 6606848 tid 1 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: finishing write tid 5 to mira0437250-398 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 751 (ObjNum 1369 snap 311 seq_num 1369) dirty exists 2014-07-01 17:09:06,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: finishing write tid 4 to mira0437250-403 2014-07-01 17:09:06,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 1425 (ObjNum 1367 snap 311 seq_num 1367) dirty exists 2014-07-01 17:09:06,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: writing mira0435601-31 from 6606848 to 7032832 tid 2 2014-07-01 17:09:06,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: writing mira0435601-31 from 7032832 to 7458816 tid 3 2014-07-01 17:09:07,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: writing mira0435601-31 from 7458816 to 7811072 tid 4 2014-07-01 17:09:07,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: read oid 86 snap -1 2014-07-01 17:09:07,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 1 to mira0435601-277 2014-07-01 17:09:07,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 2 to mira0435601-277 2014-07-01 17:09:07,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 3 to mira0435601-277 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 4 to mira0435601-277 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 1 to mira0435601-461 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 494 current snap is 0 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: seq_num 2188 ranges {4259840=536576} 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: writing mira0435601-494 from 4259840 to 4796416 tid 1 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: read oid 287 snap -1 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 3 to mira0435601-461 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 4 to mira0435601-461 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 138 (ObjNum 2181 snap 0 seq_num 2181) dirty exists 2014-07-01 17:09:07,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3558: expect (ObjNum 1776 snap 0 seq_num 1776) 2014-07-01 17:09:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2841: done (4 left) 2014-07-01 17:09:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2843: done (3 left) 2014-07-01 17:09:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2846: done (2 left) 2014-07-01 17:09:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2848: done (1 left) 2014-07-01 17:09:07,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:2849: read oid 344 snap 310 2014-07-01 17:09:07,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: write oid 357 current snap is 311 2014-07-01 17:09:07,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: seq_num 1370 ranges {444008=697805,1644007=1} 2014-07-01 17:09:07,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: writing mira0437250-357 from 444008 to 1141813 tid 1 2014-07-01 17:09:07,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: writing mira0437250-357 from 1644007 to 1644008 tid 2 2014-07-01 17:09:07,561.561 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0.589216 (8386142) 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: read oid 272 snap -1 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2849: done (3 left) 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: read oid 321 snap 278 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: write oid 335 current snap is 311 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: seq_num 1371 ranges {504165=421034,1625624=585679,2688842=215323} 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: writing mira0437250-335 from 504165 to 925199 tid 1 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: writing mira0437250-335 from 1625624 to 2211303 tid 2 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: writing mira0437250-335 from 2688842 to 2904165 tid 3 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2854: rollback oid 41 current snap is 311 2014-07-01 17:09:07,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 41 to 309 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2851: done (5 left) 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2852: done (4 left) 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: write oid 384 current snap is 311 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: seq_num 1372 ranges {447971=792838,1952236=654851,3385172=262799} 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: writing mira0437250-384 from 447971 to 1240809 tid 1 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: writing mira0437250-384 from 1952236 to 2607087 tid 2 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: writing mira0437250-384 from 3385172 to 3647971 tid 3 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: read oid 19 snap -1 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: finishing write tid 1 to mira0437250-335 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: finishing write tid 2 to mira0437250-335 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: finishing write tid 3 to mira0437250-335 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: finishing write tid 5 to mira0437250-335 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: finishing write tid 6 to mira0437250-335 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 1014 (ObjNum 1371 snap 311 seq_num 1371) dirty exists 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2853: done (5 left) 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2854: done (4 left) 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: rollback oid 151 current snap is 311 2014-07-01 17:09:07,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 151 to 300 2014-07-01 17:09:07,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: snap_create 2014-07-01 17:09:07,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 151 v 1388 (ObjNum 150 snap 0 seq_num 150) dirty exists 2014-07-01 17:09:07,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: expect (ObjNum 1172 snap 246 seq_num 1172) 2014-07-01 17:09:08,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2856: done (5 left) 2014-07-01 17:09:08,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2857: done (4 left) 2014-07-01 17:09:08,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2858: done (3 left) 2014-07-01 17:09:08,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: delete oid 179 current snap is 312 2014-07-01 17:09:08,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2859: done (3 left) 2014-07-01 17:09:08,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: write oid 234 current snap is 312 2014-07-01 17:09:08,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: seq_num 1374 ranges {88029=1} 2014-07-01 17:09:08,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: writing mira0437250-234 from 88029 to 88030 tid 1 2014-07-01 17:09:08,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: write oid 330 current snap is 312 2014-07-01 17:09:08,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: seq_num 1375 ranges {549212=495771,1349211=1} 2014-07-01 17:09:08,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: writing mira0437250-330 from 549212 to 1044983 tid 1 2014-07-01 17:09:08,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: writing mira0437250-330 from 1349211 to 1349212 tid 2 2014-07-01 17:09:08,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: write oid 201 current snap is 312 2014-07-01 17:09:08,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: seq_num 1376 ranges {645466=400000} 2014-07-01 17:09:08,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: writing mira0437250-201 from 645466 to 1045466 tid 1 2014-07-01 17:09:08,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:2863: delete oid 157 current snap is 312 2014-07-01 17:09:11,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:3549: expect (ObjNum 1927 snap 0 seq_num 1927) 2014-07-01 17:09:11,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 1 to mira0437250-384 2014-07-01 17:09:11,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 2 to mira0437250-384 2014-07-01 17:09:11,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 3 to mira0437250-384 2014-07-01 17:09:11,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 5 to mira0437250-384 2014-07-01 17:09:11,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: finishing write tid 1 to mira0437250-330 2014-07-01 17:09:11,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: finishing write tid 2 to mira0437250-330 2014-07-01 17:09:11,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: finishing write tid 4 to mira0437250-330 2014-07-01 17:09:11,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: finishing write tid 1 to mira0437250-357 2014-07-01 17:09:11,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: finishing write tid 2 to mira0437250-357 2014-07-01 17:09:11,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: finishing write tid 4 to mira0437250-357 2014-07-01 17:09:11,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: finishing write tid 6 to mira0437250-384 2014-07-01 17:09:11,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 1789 (ObjNum 1372 snap 311 seq_num 1372) dirty exists 2014-07-01 17:09:11,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: finishing write tid 1 to mira0437250-201 2014-07-01 17:09:11,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: finishing write tid 3 to mira0437250-201 2014-07-01 17:09:11,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: finishing write tid 5 to mira0437250-330 2014-07-01 17:09:11,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 878 (ObjNum 1375 snap 312 seq_num 1375) dirty exists 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: finishing write tid 5 to mira0437250-357 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 1515 (ObjNum 1370 snap 311 seq_num 1370) dirty exists 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2850: done (6 left) 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2855: done (5 left) 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2861: done (4 left) 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2863: done (3 left) 2014-07-01 17:09:11,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: read oid 60 snap -1 2014-07-01 17:09:11,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: write oid 80 current snap is 312 2014-07-01 17:09:11,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: seq_num 1378 ranges {534101=695856,1746660=462518,2534100=1} 2014-07-01 17:09:11,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: writing mira0437250-80 from 534101 to 1229957 tid 1 2014-07-01 17:09:11,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: writing mira0437250-80 from 1746660 to 2209178 tid 2 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: writing mira0437250-80 from 2534100 to 2534101 tid 3 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2866: rollback oid 212 current snap is 312 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 212 to 286 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: finishing write tid 4 to mira0437250-201 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 201 v 1402 (ObjNum 1376 snap 312 seq_num 1376) dirty exists 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2862: done (5 left) 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: write oid 285 current snap is 312 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: seq_num 1379 ranges {628307=419084,1610168=618139} 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: writing mira0437250-285 from 628307 to 1047391 tid 1 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: writing mira0437250-285 from 1610168 to 2228307 tid 2 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2868: snap_create 2014-07-01 17:09:11,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 1 to mira0437250-234 2014-07-01 17:09:11,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 3 to mira0437250-234 2014-07-01 17:09:11,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: finishing write tid 1 to mira0437250-285 2014-07-01 17:09:12,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: finishing write tid 2 to mira0437250-285 2014-07-01 17:09:12,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: finishing write tid 4 to mira0437250-285 2014-07-01 17:09:12,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: finishing write tid 5 to mira0437250-285 2014-07-01 17:09:12,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 1230 (ObjNum 1379 snap 312 seq_num 1379) dirty exists 2014-07-01 17:09:12,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: expect (ObjNum 59 snap 0 seq_num 59) 2014-07-01 17:09:12,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 1037 (ObjNum 1198 snap 254 seq_num 1198) dirty exists 2014-07-01 17:09:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: finishing write tid 1 to mira0437250-80 2014-07-01 17:09:12,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: finishing write tid 2 to mira0437250-80 2014-07-01 17:09:12,390.390 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: finishing write tid 3 to mira0437250-80 2014-07-01 17:09:12,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: finishing write tid 5 to mira0437250-80 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: finishing write tid 6 to mira0437250-80 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 80 v 1113 (ObjNum 1378 snap 312 seq_num 1378) dirty exists 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: finishing write tid 4 to mira0437250-234 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 888 (ObjNum 1374 snap 312 seq_num 1374) dirty exists 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 1 to mira0437250-213 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 3 to mira0437250-213 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: finishing write tid 4 to mira0437250-213 2014-07-01 17:09:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 1450 (ObjNum 1366 snap 311 seq_num 1366) dirty exists 2014-07-01 17:09:12,573.573 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:09:12,573.573 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 17:09:12,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2838: done (6 left) 2014-07-01 17:09:12,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2860: done (5 left) 2014-07-01 17:09:12,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:2864: done (4 left) 2014-07-01 17:09:12,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2865: done (3 left) 2014-07-01 17:09:12,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2866: done (2 left) 2014-07-01 17:09:12,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2867: done (1 left) 2014-07-01 17:09:12,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2868: done (0 left) 2014-07-01 17:09:12,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:2869: rollback oid 340 current snap is 313 2014-07-01 17:09:12,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 340 to 291 2014-07-01 17:09:12,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: read oid 166 snap -1 2014-07-01 17:09:12,912.912 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: read oid 450 snap -1 2014-07-01 17:09:13,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2872: delete oid 146 current snap is 313 2014-07-01 17:09:13,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: expect (ObjNum 1169 snap 246 seq_num 1169) 2014-07-01 17:09:13,983.983 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 17:09:13,996.996 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 17:09:14,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: expect (ObjNum 449 snap 0 seq_num 449) 2014-07-01 17:09:14,986.986 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 17:09:14,995.995 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 17:09:16,000.000 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 17:09:16,014.014 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 17:09:16,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 936 (ObjNum 34673744 snap 0 seq_num 17143248) dirty dne 2014-07-01 17:09:16,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2869: done (3 left) 2014-07-01 17:09:16,206.206 INFO:teuthology.task.rados.rados.0.mira043.stdout:2870: done (2 left) 2014-07-01 17:09:16,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2871: done (1 left) 2014-07-01 17:09:16,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2872: done (0 left) 2014-07-01 17:09:16,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: write oid 172 current snap is 313 2014-07-01 17:09:16,207.207 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: seq_num 1381 ranges {234318=1} 2014-07-01 17:09:16,208.208 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: writing mira0437250-172 from 234318 to 234319 tid 1 2014-07-01 17:09:16,209.209 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: delete oid 139 current snap is 313 2014-07-01 17:09:16,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2874: done (1 left) 2014-07-01 17:09:16,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: read oid 499 snap -1 2014-07-01 17:09:16,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:2876: snap_remove snap 311 2014-07-01 17:09:16,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2875: done (2 left) 2014-07-01 17:09:16,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2876: done (1 left) 2014-07-01 17:09:16,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: read oid 420 snap -1 2014-07-01 17:09:16,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: write oid 430 current snap is 313 2014-07-01 17:09:16,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: seq_num 1383 ranges {425802=493822,1347977=716342,2629051=530682,3625801=1} 2014-07-01 17:09:16,972.972 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 17:09:16,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: writing mira0437250-430 from 425802 to 919624 tid 1 2014-07-01 17:09:16,999.999 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 17:09:17,005.005 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: writing mira0437250-430 from 1347977 to 2064319 tid 2 2014-07-01 17:09:17,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: writing mira0437250-430 from 2629051 to 3159733 tid 3 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: writing mira0437250-430 from 3625801 to 3625802 tid 4 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: write oid 470 current snap is 313 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: seq_num 1384 ranges {734536=466650,1924190=10346} 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: writing mira0437250-470 from 734536 to 1201186 tid 1 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: writing mira0437250-470 from 1924190 to 1934536 tid 2 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: write oid 157 current snap is 313 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: seq_num 1385 ranges {430790=473527,1388723=242067} 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: writing mira0437250-157 from 430790 to 904317 tid 1 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: writing mira0437250-157 from 1388723 to 1630790 tid 2 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: read oid 396 snap -1 2014-07-01 17:09:17,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:2877: done (5 left) 2014-07-01 17:09:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: read oid 177 snap -1 2014-07-01 17:09:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:2883: snap_create 2014-07-01 17:09:17,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: expect (ObjNum 1920 snap 0 seq_num 1920) 2014-07-01 17:09:19,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2883: done (6 left) 2014-07-01 17:09:19,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: delete oid 448 current snap is 314 2014-07-01 17:09:20,007.007 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 1 to mira0435601-116 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 3 to mira0435601-116 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 1 to mira0435601-393 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 2 to mira0435601-393 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 4 to mira0435601-116 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 116 v 135 (ObjNum 2182 snap 0 seq_num 2182) dirty exists 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 3 to mira0435601-393 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 4 to mira0435601-393 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: done (13 left) 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3549: done (12 left) 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: done (11 left) 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: done (10 left) 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3558: done (9 left) 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: read oid 205 snap -1 2014-07-01 17:09:20,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 5 to mira0435601-393 2014-07-01 17:09:20,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 7 to mira0435601-393 2014-07-01 17:09:20,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 1 to mira0435601-31 2014-07-01 17:09:20,019.019 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 17:09:20,029.029 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 482 current snap is 0 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: seq_num 2189 ranges {1015808=565248,1581056=487424,2068480=716800} 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0435601-482 from 1015808 to 1581056 tid 1 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0435601-482 from 1581056 to 2068480 tid 2 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0435601-482 from 2068480 to 2785280 tid 3 2014-07-01 17:09:20,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3566: read oid 162 snap -1 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3567: read oid 230 snap -1 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: finishing write tid 8 to mira0435601-393 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 393 v 195 (ObjNum 2186 snap 0 seq_num 2186) dirty exists 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 2 to mira0435601-31 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 3 to mira0435601-31 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 4 to mira0435601-31 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 6 to mira0435601-31 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: finishing write tid 7 to mira0435601-31 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 344 (ObjNum 2187 snap 0 seq_num 2187) dirty exists 2014-07-01 17:09:20,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: expect (ObjNum 1569 snap 0 seq_num 1569) 2014-07-01 17:09:20,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: finishing write tid 1 to mira0437250-172 2014-07-01 17:09:20,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2884: done (6 left) 2014-07-01 17:09:20,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2885: rollback oid 362 current snap is 314 2014-07-01 17:09:20,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 362 to 299 2014-07-01 17:09:20,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:2886: snap_create 2014-07-01 17:09:20,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: finishing write tid 1 to mira0437250-157 2014-07-01 17:09:20,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: finishing write tid 2 to mira0437250-157 2014-07-01 17:09:20,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: finishing write tid 4 to mira0437250-157 2014-07-01 17:09:20,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: finishing write tid 5 to mira0437250-157 2014-07-01 17:09:20,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 1467 (ObjNum 1385 snap 313 seq_num 1385) dirty exists 2014-07-01 17:09:20,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 1 to mira0437250-430 2014-07-01 17:09:20,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 2 to mira0437250-430 2014-07-01 17:09:20,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: expect (ObjNum 1160 snap 238 seq_num 1160) 2014-07-01 17:09:21,142.142 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 17:09:21,151.151 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 17:09:21,151.151 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:21,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 3 to mira0437250-430 2014-07-01 17:09:21,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 4 to mira0437250-430 2014-07-01 17:09:21,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 6 to mira0437250-430 2014-07-01 17:09:21,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: finishing write tid 3 to mira0437250-172 2014-07-01 17:09:21,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: finishing write tid 4 to mira0437250-172 2014-07-01 17:09:21,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 1485 (ObjNum 1381 snap 313 seq_num 1381) dirty exists 2014-07-01 17:09:21,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: finishing write tid 7 to mira0437250-430 2014-07-01 17:09:21,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 1032 (ObjNum 1383 snap 313 seq_num 1383) dirty exists 2014-07-01 17:09:21,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: expect (ObjNum 519 snap 10 seq_num 519) 2014-07-01 17:09:22,043.043 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:22,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: finishing write tid 1 to mira0437250-470 2014-07-01 17:09:22,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 1271 (ObjNum 361 snap 0 seq_num 361) dirty exists 2014-07-01 17:09:22,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: finishing write tid 2 to mira0437250-470 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: finishing write tid 4 to mira0437250-470 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: finishing write tid 5 to mira0437250-470 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 470 v 773 (ObjNum 1384 snap 313 seq_num 1384) dirty exists 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2873: done (7 left) 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2878: done (6 left) 2014-07-01 17:09:22,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:2879: done (5 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2880: done (4 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2881: done (3 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2882: done (2 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2885: done (1 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2886: done (0 left) 2014-07-01 17:09:22,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: delete oid 380 current snap is 315 2014-07-01 17:09:22,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:2887: done (0 left) 2014-07-01 17:09:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: write oid 53 current snap is 315 2014-07-01 17:09:22,855.855 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: seq_num 1388 ranges {414655=541388,1543599=702390,2664555=150100} 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: writing mira0437250-53 from 414655 to 956043 tid 1 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: writing mira0437250-53 from 1543599 to 2245989 tid 2 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: writing mira0437250-53 from 2664555 to 2814655 tid 3 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: read oid 290 snap 305 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: finishing write tid 1 to mira0437250-53 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2890: snap_create 2014-07-01 17:09:22,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 17:09:22,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: finishing write tid 2 to mira0437250-53 2014-07-01 17:09:22,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: finishing write tid 3 to mira0437250-53 2014-07-01 17:09:22,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: finishing write tid 5 to mira0437250-53 2014-07-01 17:09:22,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: finishing write tid 6 to mira0437250-53 2014-07-01 17:09:22,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 1037 (ObjNum 1388 snap 315 seq_num 1388) dirty exists 2014-07-01 17:09:23,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2888: done (2 left) 2014-07-01 17:09:23,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:2889: done (1 left) 2014-07-01 17:09:23,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2890: done (0 left) 2014-07-01 17:09:23,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: write oid 461 current snap is 316 2014-07-01 17:09:23,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: seq_num 1389 ranges {626237=796840,1905283=524009,2864756=161481} 2014-07-01 17:09:23,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: writing mira0437250-461 from 626237 to 1423077 tid 1 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: writing mira0437250-461 from 1905283 to 2429292 tid 2 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: writing mira0437250-461 from 2864756 to 3026237 tid 3 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: read oid 354 snap -1 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: write oid 40 current snap is 316 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: seq_num 1390 ranges {419642=400000} 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: writing mira0437250-40 from 419642 to 819642 tid 1 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: read oid 474 snap -1 2014-07-01 17:09:23,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2895: snap_remove snap 300 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: finishing write tid 1 to mira0437250-461 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: finishing write tid 2 to mira0437250-461 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: finishing write tid 3 to mira0437250-461 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: finishing write tid 5 to mira0437250-461 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: finishing write tid 6 to mira0437250-461 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 461 v 812 (ObjNum 1389 snap 316 seq_num 1389) dirty exists 2014-07-01 17:09:23,195.195 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: expect (ObjNum 822 snap 130 seq_num 822) 2014-07-01 17:09:23,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: expect (ObjNum 2056 snap 0 seq_num 2056) 2014-07-01 17:09:24,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:2891: done (4 left) 2014-07-01 17:09:24,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:2894: done (3 left) 2014-07-01 17:09:24,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:2895: done (2 left) 2014-07-01 17:09:24,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:2896: snap_create 2014-07-01 17:09:24,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: expect (ObjNum 1069 snap 0 seq_num 1069) 2014-07-01 17:09:25,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2896: done (2 left) 2014-07-01 17:09:25,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: rollback oid 243 current snap is 317 2014-07-01 17:09:25,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 243 to 316 2014-07-01 17:09:25,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: write oid 251 current snap is 317 2014-07-01 17:09:25,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: seq_num 1391 ranges {515895=1} 2014-07-01 17:09:25,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: writing mira0437250-251 from 515895 to 515896 tid 1 2014-07-01 17:09:25,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: read oid 102 snap -1 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: snap_remove snap 245 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: finishing write tid 1 to mira0437250-251 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: finishing write tid 3 to mira0437250-251 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: finishing write tid 4 to mira0437250-251 2014-07-01 17:09:25,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 819 (ObjNum 1391 snap 317 seq_num 1391) dirty exists 2014-07-01 17:09:25,454.454 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:25,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 1 to mira0437250-40 2014-07-01 17:09:25,949.949 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 3 to mira0437250-40 2014-07-01 17:09:26,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: finishing write tid 4 to mira0437250-40 2014-07-01 17:09:26,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 40 v 1065 (ObjNum 1390 snap 316 seq_num 1390) dirty exists 2014-07-01 17:09:26,209.209 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:26,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 6 to mira0435601-277 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 7 to mira0435601-277 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 160 (ObjNum 2184 snap 0 seq_num 2184) dirty exists 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: finishing write tid 1 to mira0435601-247 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: finishing write tid 2 to mira0435601-247 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: finishing write tid 4 to mira0435601-247 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: finishing write tid 5 to mira0435601-247 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 304 (ObjNum 2185 snap 0 seq_num 2185) dirty exists 2014-07-01 17:09:26,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: expect (ObjNum 1968 snap 0 seq_num 1968) 2014-07-01 17:09:26,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2893: done (5 left) 2014-07-01 17:09:26,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2897: done (4 left) 2014-07-01 17:09:26,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2898: done (3 left) 2014-07-01 17:09:26,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2900: done (2 left) 2014-07-01 17:09:26,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2901: delete oid 190 current snap is 317 2014-07-01 17:09:26,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: expect (ObjNum 1181 snap 250 seq_num 1181) 2014-07-01 17:09:26,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: expect (ObjNum 731 snap 99 seq_num 731) 2014-07-01 17:09:27,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2892: done (2 left) 2014-07-01 17:09:27,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:2899: done (1 left) 2014-07-01 17:09:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:2901: done (0 left) 2014-07-01 17:09:27,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:2902: delete oid 342 current snap is 317 2014-07-01 17:09:28,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: finishing write tid 1 to mira0435601-494 2014-07-01 17:09:28,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: finishing write tid 3 to mira0435601-494 2014-07-01 17:09:28,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: finishing write tid 4 to mira0435601-494 2014-07-01 17:09:28,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 306 (ObjNum 2188 snap 0 seq_num 2188) dirty exists 2014-07-01 17:09:28,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: done (12 left) 2014-07-01 17:09:28,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: done (11 left) 2014-07-01 17:09:28,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: done (10 left) 2014-07-01 17:09:28,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: done (9 left) 2014-07-01 17:09:28,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: done (8 left) 2014-07-01 17:09:28,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: done (7 left) 2014-07-01 17:09:28,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: done (6 left) 2014-07-01 17:09:28,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: done (5 left) 2014-07-01 17:09:28,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: done (4 left) 2014-07-01 17:09:28,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 297 current snap is 0 2014-07-01 17:09:28,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: seq_num 2190 ranges {0=282624} 2014-07-01 17:09:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: writing mira0435601-297 from 0 to 282624 tid 1 2014-07-01 17:09:28,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: read oid 104 snap -1 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 382 current snap is 0 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: seq_num 2191 ranges {0=757760,757760=651264,1409024=475136,1884160=479232} 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: writing mira0435601-382 from 0 to 757760 tid 1 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: writing mira0435601-382 from 757760 to 1409024 tid 2 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: writing mira0435601-382 from 1409024 to 1884160 tid 3 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: writing mira0435601-382 from 1884160 to 2363392 tid 4 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 322 current snap is 0 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: seq_num 2192 ranges {909312=786432,1695744=401408} 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: writing mira0435601-322 from 909312 to 1695744 tid 1 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: writing mira0435601-322 from 1695744 to 2097152 tid 2 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3572: read oid 44 snap -1 2014-07-01 17:09:28,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: expect (ObjNum 204 snap 0 seq_num 204) 2014-07-01 17:09:29,586.586 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 1 to mira0435601-482 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 2 to mira0435601-482 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 3 to mira0435601-482 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 5 to mira0435601-482 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 6 to mira0435601-482 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 283 (ObjNum 2189 snap 0 seq_num 2189) dirty exists 2014-07-01 17:09:29,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3566: expect (ObjNum 1597 snap 0 seq_num 1597) 2014-07-01 17:09:30,298.298 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 12, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.009970', u'skew': u'0.001400', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.005155', u'skew': u'0.001407', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:08:45.734346', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 879194576, u'health': u'HEALTH_OK', u'kb_used': 17586476, u'store_stats': {u'bytes_total': 11839925, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4499877}}, {u'last_updated': u'2014-07-01 08:08:56.012022', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883713140, u'health': u'HEALTH_OK', u'kb_used': 13067912, u'store_stats': {u'bytes_total': 17081550, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498622}}, {u'last_updated': u'2014-07-01 08:09:13.316215', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883613824, u'health': u'HEALTH_OK', u'kb_used': 13167228, u'store_stats': {u'bytes_total': 19178718, u'bytes_log': 14614528, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498638}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'15 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'12 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'11 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 295/2899 objects degraded (10.176%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 66 > pgp_num 56'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 240082455, u'degraded_objects': 295, u'num_pgs': 524, u'recovering_keys_per_sec': 113, u'data_bytes': 2481489256, u'degraded_total': 2899, u'bytes_used': 5924261888, u'recovering_objects_per_sec': 113, u'write_bytes_sec': 440351266, u'op_per_sec': 1052, u'version': 2734, u'pgs_by_state': [{u'count': 12, u'state_name': u'active+recovery_wait'}, {u'count': 497, u'state_name': u'active+clean'}, {u'count': 15, u'state_name': u'active+recovering'}], u'read_bytes_sec': 15993791, u'degraded_ratio': u'10.176', u'bytes_avail': 5992374784000}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1318, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:09:30,298.298 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:09:31,030.030 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 17:09:31,030.030 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:31,814.814 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:33,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: expect (ObjNum 1649 snap 0 seq_num 1649) 2014-07-01 17:09:33,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 1 to mira0435601-382 2014-07-01 17:09:33,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 2 to mira0435601-382 2014-07-01 17:09:33,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 3 to mira0435601-382 2014-07-01 17:09:33,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 4 to mira0435601-382 2014-07-01 17:09:33,246.246 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 6 to mira0435601-382 2014-07-01 17:09:33,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: finishing write tid 7 to mira0435601-382 2014-07-01 17:09:33,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 382 v 137 (ObjNum 2191 snap 0 seq_num 2191) dirty exists 2014-07-01 17:09:33,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: finishing write tid 1 to mira0435601-297 2014-07-01 17:09:33,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: finishing write tid 3 to mira0435601-297 2014-07-01 17:09:33,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: finishing write tid 1 to mira0435601-322 2014-07-01 17:09:33,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: finishing write tid 2 to mira0435601-322 2014-07-01 17:09:33,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: finishing write tid 4 to mira0435601-322 2014-07-01 17:09:33,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: done (8 left) 2014-07-01 17:09:33,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: done (7 left) 2014-07-01 17:09:33,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3566: done (6 left) 2014-07-01 17:09:33,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3567: done (5 left) 2014-07-01 17:09:33,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: done (4 left) 2014-07-01 17:09:33,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: done (3 left) 2014-07-01 17:09:33,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3572: done (2 left) 2014-07-01 17:09:33,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 206 current snap is 0 2014-07-01 17:09:33,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: seq_num 2193 ranges {4149248=503808,4653056=466944,5120000=421888,5541888=696320,6238208=20480} 2014-07-01 17:09:33,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0435601-206 from 4149248 to 4653056 tid 1 2014-07-01 17:09:33,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0435601-206 from 4653056 to 5120000 tid 2 2014-07-01 17:09:33,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0435601-206 from 5120000 to 5541888 tid 3 2014-07-01 17:09:33,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0435601-206 from 5541888 to 6238208 tid 4 2014-07-01 17:09:33,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0435601-206 from 6238208 to 6258688 tid 5 2014-07-01 17:09:33,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: read oid 303 snap -1 2014-07-01 17:09:33,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: finishing write tid 4 to mira0435601-297 2014-07-01 17:09:33,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 297 v 222 (ObjNum 2190 snap 0 seq_num 2190) dirty exists 2014-07-01 17:09:33,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: finishing write tid 5 to mira0435601-322 2014-07-01 17:09:33,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 345 (ObjNum 2192 snap 0 seq_num 2192) dirty exists 2014-07-01 17:09:33,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 1 to mira0435601-206 2014-07-01 17:09:33,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 2 to mira0435601-206 2014-07-01 17:09:33,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 3 to mira0435601-206 2014-07-01 17:09:33,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: done (3 left) 2014-07-01 17:09:33,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: done (2 left) 2014-07-01 17:09:33,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 340 current snap is 0 2014-07-01 17:09:33,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: seq_num 2194 ranges {1658880=528384,2187264=401408} 2014-07-01 17:09:33,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: writing mira0435601-340 from 1658880 to 2187264 tid 1 2014-07-01 17:09:33,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: writing mira0435601-340 from 2187264 to 2588672 tid 2 2014-07-01 17:09:33,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: read oid 152 snap -1 2014-07-01 17:09:33,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 4 to mira0435601-206 2014-07-01 17:09:33,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: expect (ObjNum 1865 snap 0 seq_num 1865) 2014-07-01 17:09:34,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 5 to mira0435601-206 2014-07-01 17:09:34,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 7 to mira0435601-206 2014-07-01 17:09:34,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 8 to mira0435601-206 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 246 (ObjNum 2193 snap 0 seq_num 2193) dirty exists 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: finishing write tid 1 to mira0435601-340 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: finishing write tid 2 to mira0435601-340 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: finishing write tid 4 to mira0435601-340 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: finishing write tid 5 to mira0435601-340 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 340 v 249 (ObjNum 2194 snap 0 seq_num 2194) dirty exists 2014-07-01 17:09:34,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: expect (ObjNum 1740 snap 0 seq_num 1740) 2014-07-01 17:09:35,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2902: done (0 left) 2014-07-01 17:09:35,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: read oid 333 snap -1 2014-07-01 17:09:35,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: snap_remove snap 266 2014-07-01 17:09:35,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: expect (ObjNum 1357 snap 309 seq_num 1357) 2014-07-01 17:09:36,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: done (3 left) 2014-07-01 17:09:36,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: done (2 left) 2014-07-01 17:09:36,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: done (1 left) 2014-07-01 17:09:36,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: done (0 left) 2014-07-01 17:09:36,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 102 current snap is 0 2014-07-01 17:09:36,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: seq_num 2195 ranges {3690496=557056,4247552=684032,4931584=770048,5701632=151552} 2014-07-01 17:09:36,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: writing mira0435601-102 from 3690496 to 4247552 tid 1 2014-07-01 17:09:36,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: writing mira0435601-102 from 4247552 to 4931584 tid 2 2014-07-01 17:09:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: writing mira0435601-102 from 4931584 to 5701632 tid 3 2014-07-01 17:09:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: writing mira0435601-102 from 5701632 to 5853184 tid 4 2014-07-01 17:09:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: read oid 23 snap -1 2014-07-01 17:09:36,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 184 current snap is 0 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: seq_num 2196 ranges {1241088=479232} 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: writing mira0435601-184 from 1241088 to 1720320 tid 1 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: seq_num 2197 ranges {2072576=348160} 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: writing mira0435601-124 from 2072576 to 2420736 tid 1 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3581: read oid 97 snap -1 2014-07-01 17:09:36,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: expect (ObjNum 2160 snap 0 seq_num 2160) 2014-07-01 17:09:36,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:2903: done (1 left) 2014-07-01 17:09:36,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2904: done (0 left) 2014-07-01 17:09:36,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: rollback oid 213 current snap is 317 2014-07-01 17:09:36,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 213 to 312 2014-07-01 17:09:36,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: snap_create 2014-07-01 17:09:37,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:2906: done (1 left) 2014-07-01 17:09:37,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: read oid 180 snap 317 2014-07-01 17:09:37,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:2908: rollback oid 477 current snap is 318 2014-07-01 17:09:37,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 477 to 303 2014-07-01 17:09:37,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: write oid 264 current snap is 318 2014-07-01 17:09:37,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: seq_num 1394 ranges {616113=444406,1701927=514186} 2014-07-01 17:09:37,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: writing mira0437250-264 from 616113 to 1060519 tid 1 2014-07-01 17:09:37,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: writing mira0437250-264 from 1701927 to 2216113 tid 2 2014-07-01 17:09:37,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: write oid 445 current snap is 318 2014-07-01 17:09:37,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: seq_num 1395 ranges {559919=568658,1532411=447004,2382764=577155} 2014-07-01 17:09:37,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: writing mira0437250-445 from 559919 to 1128577 tid 1 2014-07-01 17:09:37,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: writing mira0437250-445 from 1532411 to 1979415 tid 2 2014-07-01 17:09:37,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: writing mira0437250-445 from 2382764 to 2959919 tid 3 2014-07-01 17:09:37,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:2911: snap_create 2014-07-01 17:09:37,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 1480 (ObjNum 1366 snap 311 seq_num 1366) dirty exists 2014-07-01 17:09:37,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 1825 (ObjNum 1312 snap 297 seq_num 1312) dirty exists 2014-07-01 17:09:37,627.627 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: finishing write tid 1 to mira0437250-264 2014-07-01 17:09:37,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: finishing write tid 2 to mira0437250-264 2014-07-01 17:09:37,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: finishing write tid 4 to mira0437250-264 2014-07-01 17:09:37,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: finishing write tid 5 to mira0437250-264 2014-07-01 17:09:37,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1475 (ObjNum 1394 snap 318 seq_num 1394) dirty exists 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 1 to mira0437250-445 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 2 to mira0437250-445 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 3 to mira0437250-445 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 5 to mira0437250-445 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: finishing write tid 6 to mira0437250-445 2014-07-01 17:09:37,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 1411 (ObjNum 1395 snap 318 seq_num 1395) dirty exists 2014-07-01 17:09:38,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 1 to mira0435601-102 2014-07-01 17:09:38,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: finishing write tid 1 to mira0435601-184 2014-07-01 17:09:38,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 2 to mira0435601-102 2014-07-01 17:09:38,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 3 to mira0435601-102 2014-07-01 17:09:38,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: finishing write tid 3 to mira0435601-184 2014-07-01 17:09:38,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 4 to mira0435601-102 2014-07-01 17:09:38,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 6 to mira0435601-102 2014-07-01 17:09:38,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: finishing write tid 1 to mira0435601-124 2014-07-01 17:09:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: finishing write tid 7 to mira0435601-102 2014-07-01 17:09:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 293 (ObjNum 2195 snap 0 seq_num 2195) dirty exists 2014-07-01 17:09:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: finishing write tid 3 to mira0435601-124 2014-07-01 17:09:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: finishing write tid 4 to mira0435601-184 2014-07-01 17:09:38,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 212 (ObjNum 2196 snap 0 seq_num 2196) dirty exists 2014-07-01 17:09:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: finishing write tid 4 to mira0435601-124 2014-07-01 17:09:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 341 (ObjNum 2197 snap 0 seq_num 2197) dirty exists 2014-07-01 17:09:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: done (4 left) 2014-07-01 17:09:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: done (3 left) 2014-07-01 17:09:38,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: done (2 left) 2014-07-01 17:09:38,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: done (1 left) 2014-07-01 17:09:38,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3581: done (0 left) 2014-07-01 17:09:38,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 383 current snap is 0 2014-07-01 17:09:38,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: seq_num 2198 ranges {3141632=221184} 2014-07-01 17:09:38,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: writing mira0435601-383 from 3141632 to 3362816 tid 1 2014-07-01 17:09:38,138.138 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:38,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 390 current snap is 0 2014-07-01 17:09:38,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: seq_num 2199 ranges {77824=651264,729088=663552,1392640=139264} 2014-07-01 17:09:38,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0435601-390 from 77824 to 729088 tid 1 2014-07-01 17:09:38,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0435601-390 from 729088 to 1392640 tid 2 2014-07-01 17:09:38,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0435601-390 from 1392640 to 1531904 tid 3 2014-07-01 17:09:38,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3584: delete oid 75 current snap is 0 2014-07-01 17:09:38,307.307 INFO:teuthology.task.rados.rados.0.mira043.stdout:3584: done (2 left) 2014-07-01 17:09:38,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 170 current snap is 0 2014-07-01 17:09:38,308.308 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: seq_num 2201 ranges {6672384=442368,7114752=675840,7790592=561152,8351744=368640} 2014-07-01 17:09:38,365.365 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: writing mira0435601-170 from 6672384 to 7114752 tid 1 2014-07-01 17:09:38,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2905: done (5 left) 2014-07-01 17:09:38,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2907: done (4 left) 2014-07-01 17:09:38,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2908: done (3 left) 2014-07-01 17:09:38,366.366 INFO:teuthology.task.rados.rados.0.mira043.stdout:2909: done (2 left) 2014-07-01 17:09:38,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:2910: done (1 left) 2014-07-01 17:09:38,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2911: done (0 left) 2014-07-01 17:09:38,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:2912: rollback oid 253 current snap is 319 2014-07-01 17:09:38,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 253 to 310 2014-07-01 17:09:38,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: write oid 95 current snap is 319 2014-07-01 17:09:38,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: seq_num 1396 ranges {656173=540462,1895119=552271,2656172=1} 2014-07-01 17:09:38,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: writing mira0437250-95 from 656173 to 1196635 tid 1 2014-07-01 17:09:38,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: writing mira0437250-95 from 1895119 to 2447390 tid 2 2014-07-01 17:09:38,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: writing mira0437250-95 from 2656172 to 2656173 tid 3 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: write oid 94 current snap is 319 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: seq_num 1397 ranges {599978=547339,1608227=622466,2952510=447468} 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: writing mira0435601-170 from 7114752 to 7790592 tid 2 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: writing mira0435601-170 from 7790592 to 8351744 tid 3 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: writing mira0435601-170 from 8351744 to 8720384 tid 4 2014-07-01 17:09:38,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3586: read oid 182 snap -1 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: finishing write tid 1 to mira0435601-383 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: finishing write tid 3 to mira0435601-383 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 1 to mira0435601-390 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 2 to mira0435601-390 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: read oid 234 snap -1 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 3 to mira0435601-390 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 5 to mira0435601-390 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: finishing write tid 4 to mira0435601-383 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 383 v 135 (ObjNum 2198 snap 0 seq_num 2198) dirty exists 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 6 to mira0435601-390 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 390 v 178 (ObjNum 2199 snap 0 seq_num 2199) dirty exists 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 1 to mira0435601-170 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: done (4 left) 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: done (3 left) 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 83 current snap is 0 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: seq_num 2202 ranges {2469888=471040,2940928=401408} 2014-07-01 17:09:38,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: writing mira0435601-83 from 2469888 to 2940928 tid 1 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: writing mira0435601-83 from 2940928 to 3342336 tid 2 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 203 current snap is 0 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: seq_num 2203 ranges {0=741376} 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: writing mira0437250-94 from 599978 to 1147317 tid 1 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: writing mira0437250-94 from 1608227 to 2230693 tid 2 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: writing mira0437250-94 from 2952510 to 3399978 tid 3 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: write oid 464 current snap is 319 2014-07-01 17:09:38,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: seq_num 1398 ranges {661159=1} 2014-07-01 17:09:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: writing mira0437250-464 from 661159 to 661160 tid 1 2014-07-01 17:09:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: write oid 285 current snap is 319 2014-07-01 17:09:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: seq_num 1399 ranges {356389=1} 2014-07-01 17:09:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: writing mira0437250-285 from 356389 to 356390 tid 1 2014-07-01 17:09:38,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:2917: snap_create 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 253 v 1285 (ObjNum 1200 snap 254 seq_num 1200) dirty exists 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 1 to mira0437250-95 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 2 to mira0437250-95 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 3 to mira0437250-95 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 5 to mira0437250-95 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: finishing write tid 6 to mira0437250-95 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 899 (ObjNum 1396 snap 319 seq_num 1396) dirty exists 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: finishing write tid 1 to mira0437250-94 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: finishing write tid 1 to mira0437250-285 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: finishing write tid 3 to mira0437250-285 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: finishing write tid 2 to mira0437250-94 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: finishing write tid 3 to mira0437250-94 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: finishing write tid 5 to mira0437250-94 2014-07-01 17:09:38,534.534 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: writing mira0435601-203 from 0 to 741376 tid 1 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: read oid 263 snap -1 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 2 to mira0435601-170 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 3 to mira0435601-170 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 4 to mira0435601-170 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 6 to mira0435601-170 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3586: done (5 left) 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3591: delete oid 342 current snap is 0 2014-07-01 17:09:38,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: expect (ObjNum 980 snap 0 seq_num 980) 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: finishing write tid 6 to mira0437250-94 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 1480 (ObjNum 1397 snap 319 seq_num 1397) dirty exists 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: finishing write tid 1 to mira0437250-464 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: finishing write tid 3 to mira0437250-464 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: finishing write tid 4 to mira0437250-464 2014-07-01 17:09:38,641.641 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 1102 (ObjNum 1398 snap 319 seq_num 1398) dirty exists 2014-07-01 17:09:38,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: finishing write tid 4 to mira0437250-285 2014-07-01 17:09:38,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 1241 (ObjNum 1399 snap 319 seq_num 1399) dirty exists 2014-07-01 17:09:38,876.876 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:39,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2912: done (5 left) 2014-07-01 17:09:39,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2913: done (4 left) 2014-07-01 17:09:39,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2914: done (3 left) 2014-07-01 17:09:39,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:2915: done (2 left) 2014-07-01 17:09:39,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2916: done (1 left) 2014-07-01 17:09:39,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2917: done (0 left) 2014-07-01 17:09:39,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: snap_create 2014-07-01 17:09:40,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:2918: done (0 left) 2014-07-01 17:09:40,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: read oid 272 snap 299 2014-07-01 17:09:40,380.380 INFO:teuthology.task.rados.rados.0.mira043.stdout:2920: snap_create 2014-07-01 17:09:41,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: finishing write tid 7 to mira0435601-170 2014-07-01 17:09:41,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 170 v 234 (ObjNum 2201 snap 0 seq_num 2201) dirty exists 2014-07-01 17:09:41,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: finishing write tid 1 to mira0435601-203 2014-07-01 17:09:41,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: finishing write tid 3 to mira0435601-203 2014-07-01 17:09:41,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: finishing write tid 4 to mira0435601-203 2014-07-01 17:09:41,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 203 v 378 (ObjNum 2203 snap 0 seq_num 2203) dirty exists 2014-07-01 17:09:41,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: expect (ObjNum 1991 snap 0 seq_num 1991) 2014-07-01 17:09:41,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2919: done (1 left) 2014-07-01 17:09:41,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2920: done (0 left) 2014-07-01 17:09:41,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: write oid 10 current snap is 322 2014-07-01 17:09:41,389.389 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: seq_num 1400 ranges {701220=400000} 2014-07-01 17:09:41,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: writing mira0437250-10 from 701220 to 1101220 tid 1 2014-07-01 17:09:41,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: read oid 229 snap 307 2014-07-01 17:09:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: write oid 126 current snap is 322 2014-07-01 17:09:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: seq_num 1401 ranges {646049=470652,1725812=120237} 2014-07-01 17:09:41,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: writing mira0437250-126 from 646049 to 1116701 tid 1 2014-07-01 17:09:41,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: writing mira0437250-126 from 1725812 to 1846049 tid 2 2014-07-01 17:09:41,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: write oid 88 current snap is 322 2014-07-01 17:09:41,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: seq_num 1402 ranges {457631=594904,1672648=384983} 2014-07-01 17:09:41,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: writing mira0437250-88 from 457631 to 1052535 tid 1 2014-07-01 17:09:41,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: writing mira0437250-88 from 1672648 to 2057631 tid 2 2014-07-01 17:09:41,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:2925: snap_create 2014-07-01 17:09:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 1 to mira0437250-10 2014-07-01 17:09:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 3 to mira0437250-10 2014-07-01 17:09:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: finishing write tid 4 to mira0437250-10 2014-07-01 17:09:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 10 v 765 (ObjNum 1400 snap 322 seq_num 1400) dirty exists 2014-07-01 17:09:41,558.558 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 1 to mira0437250-126 2014-07-01 17:09:41,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 2 to mira0437250-126 2014-07-01 17:09:41,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 4 to mira0437250-126 2014-07-01 17:09:41,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: finishing write tid 5 to mira0437250-126 2014-07-01 17:09:41,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 126 v 1460 (ObjNum 1401 snap 322 seq_num 1401) dirty exists 2014-07-01 17:09:41,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: expect (ObjNum 1082 snap 217 seq_num 1082) 2014-07-01 17:09:42,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: finishing write tid 1 to mira0437250-88 2014-07-01 17:09:42,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: finishing write tid 2 to mira0437250-88 2014-07-01 17:09:42,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: finishing write tid 4 to mira0437250-88 2014-07-01 17:09:42,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: finishing write tid 5 to mira0437250-88 2014-07-01 17:09:42,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 1517 (ObjNum 1402 snap 322 seq_num 1402) dirty exists 2014-07-01 17:09:42,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:2921: done (4 left) 2014-07-01 17:09:42,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:2922: done (3 left) 2014-07-01 17:09:42,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2923: done (2 left) 2014-07-01 17:09:42,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:2924: done (1 left) 2014-07-01 17:09:42,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2925: done (0 left) 2014-07-01 17:09:42,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2926: read oid 256 snap 320 2014-07-01 17:09:42,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: write oid 16 current snap is 323 2014-07-01 17:09:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: seq_num 1403 ranges {552861=566709,1752860=1} 2014-07-01 17:09:42,446.446 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:09:42,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: writing mira0437250-16 from 552861 to 1119570 tid 1 2014-07-01 17:09:42,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: writing mira0437250-16 from 1752860 to 1752861 tid 2 2014-07-01 17:09:42,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: read oid 262 snap -1 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: write oid 361 current snap is 323 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: seq_num 1404 ranges {745243=689937,2147441=768105,3374593=170650} 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: writing mira0437250-361 from 745243 to 1435180 tid 1 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: writing mira0437250-361 from 2147441 to 2915546 tid 2 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: writing mira0437250-361 from 3374593 to 3545243 tid 3 2014-07-01 17:09:42,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:2930: rollback oid 328 current snap is 323 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 328 to 305 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: expect (ObjNum 261 snap 0 seq_num 261) 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: finishing write tid 1 to mira0437250-16 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: finishing write tid 2 to mira0437250-16 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: finishing write tid 4 to mira0437250-16 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: finishing write tid 5 to mira0437250-16 2014-07-01 17:09:42,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 911 (ObjNum 1403 snap 323 seq_num 1403) dirty exists 2014-07-01 17:09:42,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:2926: expect (ObjNum 255 snap 0 seq_num 255) 2014-07-01 17:09:42,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 1 to mira0437250-361 2014-07-01 17:09:42,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 1414 (ObjNum 1543511072 snap 32691 seq_num 0) dirty dne 2014-07-01 17:09:42,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 2 to mira0437250-361 2014-07-01 17:09:42,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 3 to mira0437250-361 2014-07-01 17:09:42,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 5 to mira0437250-361 2014-07-01 17:09:42,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: finishing write tid 6 to mira0437250-361 2014-07-01 17:09:42,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 361 v 1290 (ObjNum 1404 snap 323 seq_num 1404) dirty exists 2014-07-01 17:09:42,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:2926: done (4 left) 2014-07-01 17:09:42,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2927: done (3 left) 2014-07-01 17:09:42,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2928: done (2 left) 2014-07-01 17:09:42,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:2929: done (1 left) 2014-07-01 17:09:42,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2930: done (0 left) 2014-07-01 17:09:42,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:2931: snap_create 2014-07-01 17:09:43,156.156 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 12, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.009970', u'skew': u'0.001400', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.005155', u'skew': u'0.001407', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:08:45.734346', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 879194576, u'health': u'HEALTH_OK', u'kb_used': 17586476, u'store_stats': {u'bytes_total': 11839925, u'bytes_log': 7274496, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4499877}}, {u'last_updated': u'2014-07-01 08:08:56.012022', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883713140, u'health': u'HEALTH_OK', u'kb_used': 13067912, u'store_stats': {u'bytes_total': 17081550, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498622}}, {u'last_updated': u'2014-07-01 08:09:13.316215', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883613824, u'health': u'HEALTH_OK', u'kb_used': 13167228, u'store_stats': {u'bytes_total': 19178718, u'bytes_log': 14614528, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498638}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 66 > pgp_num 56'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 524, u'data_bytes': 2405920167, u'bytes_used': 5560049664, u'op_per_sec': 29, u'version': 2748, u'pgs_by_state': [{u'count': 524, u'state_name': u'active+clean'}], u'read_bytes_sec': 7030850, u'write_bytes_sec': 5727551, u'bytes_avail': 5992738996224}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1325, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:09:43,157.157 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:09:43,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:2931: done (0 left) 2014-07-01 17:09:43,425.425 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: snap_create 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: finishing write tid 1 to mira0435601-83 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: finishing write tid 2 to mira0435601-83 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: finishing write tid 4 to mira0435601-83 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: finishing write tid 5 to mira0435601-83 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 259 (ObjNum 2202 snap 0 seq_num 2202) dirty exists 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: done (5 left) 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: done (4 left) 2014-07-01 17:09:43,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: done (3 left) 2014-07-01 17:09:43,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: done (2 left) 2014-07-01 17:09:43,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: done (1 left) 2014-07-01 17:09:43,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3591: done (0 left) 2014-07-01 17:09:43,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: read oid 3 snap -1 2014-07-01 17:09:43,482.482 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: read oid 148 snap -1 2014-07-01 17:09:43,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 163 current snap is 0 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: seq_num 2205 ranges {4472832=290816} 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: writing mira0435601-163 from 4472832 to 4763648 tid 1 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 220 current snap is 0 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: seq_num 2206 ranges {2265088=716800,2981888=483328,3465216=319488} 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: writing mira0435601-220 from 2265088 to 2981888 tid 1 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: writing mira0435601-220 from 2981888 to 3465216 tid 2 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: writing mira0435601-220 from 3465216 to 3784704 tid 3 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 222 current snap is 0 2014-07-01 17:09:43,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: seq_num 2207 ranges {5435392=32768} 2014-07-01 17:09:43,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: writing mira0435601-222 from 5435392 to 5468160 tid 1 2014-07-01 17:09:43,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3597: read oid 487 snap -1 2014-07-01 17:09:43,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: expect (ObjNum 2054 snap 0 seq_num 2054) 2014-07-01 17:09:43,939.939 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 17:09:43,940.940 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:44,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:2932: done (0 left) 2014-07-01 17:09:44,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:2933: rollback oid 408 current snap is 325 2014-07-01 17:09:44,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 408 to 279 2014-07-01 17:09:44,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:2934: snap_create 2014-07-01 17:09:44,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 408 v 1078 (ObjNum 407 snap 0 seq_num 407) dirty exists 2014-07-01 17:09:44,706.706 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 1 to mira0435601-163 2014-07-01 17:09:45,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 3 to mira0435601-163 2014-07-01 17:09:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 4 to mira0435601-163 2014-07-01 17:09:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 394 (ObjNum 2205 snap 0 seq_num 2205) dirty exists 2014-07-01 17:09:45,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: expect (ObjNum 992 snap 0 seq_num 992) 2014-07-01 17:09:45,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:2933: done (1 left) 2014-07-01 17:09:45,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2934: done (0 left) 2014-07-01 17:09:45,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: delete oid 62 current snap is 326 2014-07-01 17:09:45,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2935: done (0 left) 2014-07-01 17:09:45,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:2936: snap_create 2014-07-01 17:09:46,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2936: done (0 left) 2014-07-01 17:09:46,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: write oid 217 current snap is 327 2014-07-01 17:09:46,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: seq_num 1406 ranges {537750=669642,1976729=792143,2937749=1} 2014-07-01 17:09:46,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: writing mira0437250-217 from 537750 to 1207392 tid 1 2014-07-01 17:09:46,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: writing mira0437250-217 from 1976729 to 2768872 tid 2 2014-07-01 17:09:46,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: writing mira0437250-217 from 2937749 to 2937750 tid 3 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: snap_create 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: finishing write tid 1 to mira0437250-217 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: finishing write tid 2 to mira0437250-217 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: finishing write tid 3 to mira0437250-217 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: finishing write tid 5 to mira0437250-217 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: finishing write tid 6 to mira0437250-217 2014-07-01 17:09:46,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 217 v 1246 (ObjNum 1406 snap 327 seq_num 1406) dirty exists 2014-07-01 17:09:47,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2937: done (1 left) 2014-07-01 17:09:47,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2938: done (0 left) 2014-07-01 17:09:47,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:2939: snap_remove snap 322 2014-07-01 17:09:48,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2939: done (0 left) 2014-07-01 17:09:48,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: read oid 4 snap -1 2014-07-01 17:09:48,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:2941: read oid 221 snap -1 2014-07-01 17:09:48,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:2942: snap_remove snap 324 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 1 to mira0435601-220 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 2 to mira0435601-220 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 3 to mira0435601-220 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 5 to mira0435601-220 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 6 to mira0435601-220 2014-07-01 17:09:48,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 209 (ObjNum 2206 snap 0 seq_num 2206) dirty exists 2014-07-01 17:09:48,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: finishing write tid 1 to mira0435601-222 2014-07-01 17:09:48,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: finishing write tid 3 to mira0435601-222 2014-07-01 17:09:48,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: done (5 left) 2014-07-01 17:09:48,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: done (4 left) 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: done (3 left) 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: done (2 left) 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: seq_num 2208 ranges {0=458752,458752=651264,1110016=151552} 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: writing mira0435601-97 from 0 to 458752 tid 1 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: writing mira0435601-97 from 458752 to 1110016 tid 2 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: writing mira0435601-97 from 1110016 to 1261568 tid 3 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: read oid 376 snap -1 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: finishing write tid 4 to mira0435601-222 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 308 (ObjNum 2207 snap 0 seq_num 2207) dirty exists 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3597: expect (ObjNum 1806 snap 0 seq_num 1806) 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2941: expect (ObjNum 729 snap 98 seq_num 729) 2014-07-01 17:09:48,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: expect (ObjNum 3 snap 0 seq_num 3) 2014-07-01 17:09:49,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:2940: done (2 left) 2014-07-01 17:09:49,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:2941: done (1 left) 2014-07-01 17:09:49,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2942: done (0 left) 2014-07-01 17:09:49,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: delete oid 376 current snap is 328 2014-07-01 17:09:49,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2943: done (0 left) 2014-07-01 17:09:49,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:2944: snap_remove snap 309 2014-07-01 17:09:49,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: finishing write tid 1 to mira0435601-97 2014-07-01 17:09:49,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: finishing write tid 2 to mira0435601-97 2014-07-01 17:09:49,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: finishing write tid 3 to mira0435601-97 2014-07-01 17:09:49,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: finishing write tid 5 to mira0435601-97 2014-07-01 17:09:49,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: expect (ObjNum 375 snap 0 seq_num 375) 2014-07-01 17:09:50,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2944: done (0 left) 2014-07-01 17:09:50,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: read oid 31 snap -1 2014-07-01 17:09:50,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: delete oid 324 current snap is 328 2014-07-01 17:09:50,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2946: done (1 left) 2014-07-01 17:09:50,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:2947: rollback oid 316 current snap is 328 2014-07-01 17:09:50,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 316 to 317 2014-07-01 17:09:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: write oid 467 current snap is 328 2014-07-01 17:09:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: seq_num 1409 ranges {637966=1} 2014-07-01 17:09:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: writing mira0437250-467 from 637966 to 637967 tid 1 2014-07-01 17:09:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: write oid 319 current snap is 328 2014-07-01 17:09:50,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: seq_num 1410 ranges {431373=592955,1658217=424348,2431372=1} 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: writing mira0437250-319 from 431373 to 1024328 tid 1 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: writing mira0437250-319 from 1658217 to 2082565 tid 2 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: writing mira0437250-319 from 2431372 to 2431373 tid 3 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 0 (ObjNum 34725328 snap 0 seq_num 0) dirty dne 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: read oid 198 snap -1 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2947: done (4 left) 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: read oid 313 snap -1 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: snap_remove snap 327 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: finishing write tid 1 to mira0437250-467 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: finishing write tid 3 to mira0437250-467 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: finishing write tid 4 to mira0437250-467 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 1320 (ObjNum 1409 snap 328 seq_num 1409) dirty exists 2014-07-01 17:09:50,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: expect (ObjNum 30 snap 0 seq_num 30) 2014-07-01 17:09:51,029.029 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:51,609.609 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:51,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: finishing write tid 6 to mira0435601-97 2014-07-01 17:09:51,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 345 (ObjNum 2208 snap 0 seq_num 2208) dirty exists 2014-07-01 17:09:51,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: done (3 left) 2014-07-01 17:09:51,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3597: done (2 left) 2014-07-01 17:09:51,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: done (1 left) 2014-07-01 17:09:51,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: done (0 left) 2014-07-01 17:09:51,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3600: read oid 394 snap -1 2014-07-01 17:09:51,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 43 current snap is 0 2014-07-01 17:09:51,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: seq_num 2209 ranges {0=536576,536576=401408} 2014-07-01 17:09:52,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: writing mira0435601-43 from 0 to 536576 tid 1 2014-07-01 17:09:52,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: writing mira0435601-43 from 536576 to 937984 tid 2 2014-07-01 17:09:52,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 252 current snap is 0 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: seq_num 2210 ranges {8720384=655360,9375744=712704,10088448=491520} 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: writing mira0435601-252 from 8720384 to 9375744 tid 1 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: writing mira0435601-252 from 9375744 to 10088448 tid 2 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: writing mira0435601-252 from 10088448 to 10579968 tid 3 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 400 current snap is 0 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: seq_num 2211 ranges {0=679936} 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: writing mira0435601-400 from 0 to 679936 tid 1 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 17:09:52,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: seq_num 2212 ranges {0=708608,708608=475136,1183744=729088} 2014-07-01 17:09:52,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 1 to mira0437250-319 2014-07-01 17:09:52,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: expect (ObjNum 938 snap 171 seq_num 938) 2014-07-01 17:09:52,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: writing mira0435601-186 from 0 to 708608 tid 1 2014-07-01 17:09:52,164.164 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: writing mira0435601-186 from 708608 to 1183744 tid 2 2014-07-01 17:09:52,165.165 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: writing mira0435601-186 from 1183744 to 1912832 tid 3 2014-07-01 17:09:52,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3605: read oid 380 snap -1 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 1 to mira0435601-43 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 2 to mira0435601-43 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 4 to mira0435601-43 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 5 to mira0435601-43 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 169 (ObjNum 2209 snap 0 seq_num 2209) dirty exists 2014-07-01 17:09:52,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3600: expect (ObjNum 2012 snap 0 seq_num 2012) 2014-07-01 17:09:54,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: expect (ObjNum 1087 snap 218 seq_num 1087) 2014-07-01 17:09:55,200.200 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:09:55,902.902 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 12, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.009970', u'skew': u'0.001400', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.005155', u'skew': u'0.001407', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:09:45.734728', u'name': u'c', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 879070308, u'health': u'HEALTH_OK', u'kb_used': 17710744, u'store_stats': {u'bytes_total': 7107643, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5010475}}, {u'last_updated': u'2014-07-01 08:08:56.012022', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883713140, u'health': u'HEALTH_OK', u'kb_used': 13067912, u'store_stats': {u'bytes_total': 17081550, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498622}}, {u'last_updated': u'2014-07-01 08:09:13.316215', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883613824, u'health': u'HEALTH_OK', u'kb_used': 13167228, u'store_stats': {u'bytes_total': 19178718, u'bytes_log': 14614528, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4498638}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'11 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 66 > pgp_num 56'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 524, u'data_bytes': 2414313822, u'bytes_used': 5584920576, u'version': 2766, u'pgs_by_state': [{u'count': 524, u'state_name': u'active+clean'}], u'bytes_avail': 5992714125312}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1334, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:09:55,903.903 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:09:55,903.903 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:09:55,903.903 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 17:09:55,904.904 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:09:56,719.719 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:09:56,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 2 to mira0437250-319 2014-07-01 17:09:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 3 to mira0437250-319 2014-07-01 17:09:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 5 to mira0437250-319 2014-07-01 17:09:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: finishing write tid 6 to mira0437250-319 2014-07-01 17:09:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 1538 (ObjNum 1410 snap 328 seq_num 1410) dirty exists 2014-07-01 17:09:56,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:2945: done (5 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2948: done (4 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2949: done (3 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2950: done (2 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2951: done (1 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2952: done (0 left) 2014-07-01 17:09:56,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: write oid 486 current snap is 328 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: seq_num 1411 ranges {642955=564759,1887677=645967,2642954=1} 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: writing mira0437250-486 from 642955 to 1207714 tid 1 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: writing mira0437250-486 from 1887677 to 2533644 tid 2 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: writing mira0437250-486 from 2642954 to 2642955 tid 3 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2954: rollback oid 31 current snap is 328 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 31 to 312 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: finishing write tid 1 to mira0437250-486 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2955: delete oid 122 current snap is 328 2014-07-01 17:09:57,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:2955: done (2 left) 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: write oid 152 current snap is 328 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: seq_num 1413 ranges {131589=1} 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: writing mira0437250-152 from 131589 to 131590 tid 1 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: write oid 4 current snap is 328 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: seq_num 1414 ranges {626820=667693,2077626=489576,3194974=631846} 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: writing mira0437250-4 from 626820 to 1294513 tid 1 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: writing mira0437250-4 from 2077626 to 2567202 tid 2 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: writing mira0437250-4 from 3194974 to 3826820 tid 3 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: write oid 109 current snap is 328 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: seq_num 1415 ranges {723074=639497,2156686=559772,3244776=278298} 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: writing mira0437250-109 from 723074 to 1362571 tid 1 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: writing mira0437250-109 from 2156686 to 2716458 tid 2 2014-07-01 17:09:57,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: writing mira0437250-109 from 3244776 to 3523074 tid 3 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: delete oid 92 current snap is 328 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 1546 (ObjNum 30 snap 0 seq_num 30) dirty exists 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 1 to mira0437250-152 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: finishing write tid 2 to mira0437250-486 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: finishing write tid 3 to mira0437250-486 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: finishing write tid 5 to mira0437250-486 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 3 to mira0437250-152 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: finishing write tid 6 to mira0437250-486 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 960 (ObjNum 1411 snap 328 seq_num 1411) dirty exists 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: finishing write tid 4 to mira0437250-152 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 1325 (ObjNum 1413 snap 328 seq_num 1413) dirty exists 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: finishing write tid 1 to mira0437250-4 2014-07-01 17:09:57,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: finishing write tid 2 to mira0437250-4 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: finishing write tid 3 to mira0437250-4 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: finishing write tid 5 to mira0437250-4 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: finishing write tid 6 to mira0437250-4 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 4 v 1086 (ObjNum 1414 snap 328 seq_num 1414) dirty exists 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: finishing write tid 1 to mira0437250-109 2014-07-01 17:09:57,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: finishing write tid 2 to mira0437250-109 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: finishing write tid 3 to mira0437250-109 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2953: done (5 left) 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2954: done (4 left) 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2956: done (3 left) 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2957: done (2 left) 2014-07-01 17:09:57,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:2959: done (1 left) 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2960: rollback oid 460 current snap is 328 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 460 to 326 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: snap_create 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: finishing write tid 5 to mira0437250-109 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: finishing write tid 6 to mira0437250-109 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 609 (ObjNum 1415 snap 328 seq_num 1415) dirty exists 2014-07-01 17:09:57,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 460 v 1074 (ObjNum 1329 snap 306 seq_num 1329) dirty exists 2014-07-01 17:09:57,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:2958: done (2 left) 2014-07-01 17:09:57,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:2960: done (1 left) 2014-07-01 17:09:57,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:2961: done (0 left) 2014-07-01 17:09:57,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:2962: snap_create 2014-07-01 17:09:58,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2962: done (0 left) 2014-07-01 17:09:58,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: snap_create 2014-07-01 17:09:59,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:2963: done (0 left) 2014-07-01 17:09:59,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: read oid 70 snap 313 2014-07-01 17:09:59,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:2965: delete oid 77 current snap is 331 2014-07-01 17:09:59,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2965: done (1 left) 2014-07-01 17:09:59,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: snap_remove snap 310 2014-07-01 17:09:59,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: expect (ObjNum 69 snap 0 seq_num 69) 2014-07-01 17:10:00,096.096 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 17:10:00,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:2964: done (1 left) 2014-07-01 17:10:00,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:2966: done (0 left) 2014-07-01 17:10:00,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:2967: delete oid 424 current snap is 331 2014-07-01 17:10:00,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2967: done (0 left) 2014-07-01 17:10:00,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: delete oid 12 current snap is 331 2014-07-01 17:10:00,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2968: done (0 left) 2014-07-01 17:10:00,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2969: delete oid 163 current snap is 331 2014-07-01 17:10:00,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:2969: done (0 left) 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: read oid 495 snap -1 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: write oid 427 current snap is 331 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: seq_num 1421 ranges {524532=658867,1631367=483791,2763808=160724} 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: writing mira0437250-427 from 524532 to 1183399 tid 1 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: writing mira0437250-427 from 1631367 to 2115158 tid 2 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: writing mira0437250-427 from 2763808 to 2924532 tid 3 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: read oid 230 snap 307 2014-07-01 17:10:00,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: expect (ObjNum 827 snap 132 seq_num 827) 2014-07-01 17:10:00,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 1 to mira0437250-427 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 2 to mira0437250-427 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 3 to mira0437250-427 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 5 to mira0437250-427 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: finishing write tid 6 to mira0437250-427 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 427 v 1559 (ObjNum 1421 snap 331 seq_num 1421) dirty exists 2014-07-01 17:10:00,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: expect (ObjNum 692 snap 88 seq_num 692) 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: finishing write tid 1 to mira0435601-252 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: finishing write tid 2 to mira0435601-252 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: finishing write tid 3 to mira0435601-252 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: finishing write tid 5 to mira0435601-252 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: finishing write tid 6 to mira0435601-252 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 373 (ObjNum 2210 snap 0 seq_num 2210) dirty exists 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 1 to mira0435601-400 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 3 to mira0435601-400 2014-07-01 17:10:01,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 4 to mira0435601-400 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 400 v 347 (ObjNum 2211 snap 0 seq_num 2211) dirty exists 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: finishing write tid 1 to mira0435601-186 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: finishing write tid 2 to mira0435601-186 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: finishing write tid 3 to mira0435601-186 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3600: done (5 left) 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: done (4 left) 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: done (3 left) 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: done (2 left) 2014-07-01 17:10:01,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3606: delete oid 458 current snap is 0 2014-07-01 17:10:01,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: finishing write tid 5 to mira0435601-186 2014-07-01 17:10:01,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3605: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 17:10:01,657.657 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 17:10:02,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: finishing write tid 6 to mira0435601-186 2014-07-01 17:10:02,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 208 (ObjNum 2212 snap 0 seq_num 2212) dirty exists 2014-07-01 17:10:02,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: done (2 left) 2014-07-01 17:10:02,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3605: done (1 left) 2014-07-01 17:10:02,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3606: done (0 left) 2014-07-01 17:10:02,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3607: delete oid 95 current snap is 0 2014-07-01 17:10:02,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3607: done (0 left) 2014-07-01 17:10:02,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 176 current snap is 0 2014-07-01 17:10:02,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: seq_num 2215 ranges {8019968=528384,8548352=675840,9224192=126976} 2014-07-01 17:10:02,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: writing mira0435601-176 from 8019968 to 8548352 tid 1 2014-07-01 17:10:02,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: writing mira0435601-176 from 8548352 to 9224192 tid 2 2014-07-01 17:10:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: writing mira0435601-176 from 9224192 to 9351168 tid 3 2014-07-01 17:10:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 230 current snap is 0 2014-07-01 17:10:02,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: seq_num 2216 ranges {0=647168,647168=454656,1101824=675840,1777664=475136} 2014-07-01 17:10:02,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: writing mira0435601-230 from 0 to 647168 tid 1 2014-07-01 17:10:02,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: writing mira0435601-230 from 647168 to 1101824 tid 2 2014-07-01 17:10:02,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: writing mira0435601-230 from 1101824 to 1777664 tid 3 2014-07-01 17:10:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: writing mira0435601-230 from 1777664 to 2252800 tid 4 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: read oid 367 snap -1 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: finishing write tid 1 to mira0435601-176 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: finishing write tid 2 to mira0435601-176 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: finishing write tid 3 to mira0435601-176 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: finishing write tid 5 to mira0435601-176 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: finishing write tid 6 to mira0435601-176 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 264 (ObjNum 2215 snap 0 seq_num 2215) dirty exists 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 1 to mira0435601-230 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 2 to mira0435601-230 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: done (2 left) 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: read oid 408 snap -1 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3612: read oid 70 snap -1 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: read oid 185 snap -1 2014-07-01 17:10:02,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 75 current snap is 0 2014-07-01 17:10:02,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: seq_num 2217 ranges {0=671744,671744=401408} 2014-07-01 17:10:02,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: writing mira0435601-75 from 0 to 671744 tid 1 2014-07-01 17:10:02,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: writing mira0435601-75 from 671744 to 1073152 tid 2 2014-07-01 17:10:02,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 476 current snap is 0 2014-07-01 17:10:02,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: seq_num 2218 ranges {0=749568} 2014-07-01 17:10:02,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: writing mira0435601-476 from 0 to 749568 tid 1 2014-07-01 17:10:02,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3616: read oid 174 snap -1 2014-07-01 17:10:02,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 3 to mira0435601-230 2014-07-01 17:10:02,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: expect (ObjNum 2142 snap 0 seq_num 2142) 2014-07-01 17:10:02,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:2970: done (2 left) 2014-07-01 17:10:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2971: done (1 left) 2014-07-01 17:10:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2972: done (0 left) 2014-07-01 17:10:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: write oid 43 current snap is 331 2014-07-01 17:10:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: seq_num 1422 ranges {468338=665743,1744475=553986,2964009=499513,3668337=1} 2014-07-01 17:10:02,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: writing mira0437250-43 from 468338 to 1134081 tid 1 2014-07-01 17:10:02,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: writing mira0437250-43 from 1744475 to 2298461 tid 2 2014-07-01 17:10:02,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: writing mira0437250-43 from 2964009 to 3463522 tid 3 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: writing mira0437250-43 from 3668337 to 3668338 tid 4 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2974: snap_remove snap 286 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 1 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 2 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 3 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 4 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 6 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: finishing write tid 7 to mira0437250-43 2014-07-01 17:10:02,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 43 v 1471 (ObjNum 1422 snap 331 seq_num 1422) dirty exists 2014-07-01 17:10:03,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: expect (ObjNum 1933 snap 0 seq_num 1933) 2014-07-01 17:10:03,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:2973: done (1 left) 2014-07-01 17:10:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2974: done (0 left) 2014-07-01 17:10:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: write oid 426 current snap is 331 2014-07-01 17:10:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: seq_num 1423 ranges {413167=1} 2014-07-01 17:10:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: writing mira0437250-426 from 413167 to 413168 tid 1 2014-07-01 17:10:03,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: write oid 399 current snap is 331 2014-07-01 17:10:03,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: seq_num 1424 ranges {108396=1} 2014-07-01 17:10:03,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: writing mira0437250-399 from 108396 to 108397 tid 1 2014-07-01 17:10:03,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: delete oid 79 current snap is 331 2014-07-01 17:10:03,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: finishing write tid 1 to mira0437250-426 2014-07-01 17:10:03,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 1 to mira0437250-399 2014-07-01 17:10:03,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: finishing write tid 3 to mira0437250-426 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: finishing write tid 4 to mira0437250-426 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 426 v 1073 (ObjNum 1423 snap 331 seq_num 1423) dirty exists 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 3 to mira0437250-399 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: finishing write tid 4 to mira0437250-399 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 399 v 799 (ObjNum 1424 snap 331 seq_num 1424) dirty exists 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2975: done (2 left) 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2976: done (1 left) 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2977: done (0 left) 2014-07-01 17:10:03,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: snap_create 2014-07-01 17:10:04,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2978: done (0 left) 2014-07-01 17:10:04,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: snap_create 2014-07-01 17:10:05,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:2979: done (0 left) 2014-07-01 17:10:05,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: read oid 277 snap -1 2014-07-01 17:10:05,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: write oid 226 current snap is 333 2014-07-01 17:10:05,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: seq_num 1426 ranges {514409=741505,2014508=99901} 2014-07-01 17:10:05,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0437250-226 from 514409 to 1255914 tid 1 2014-07-01 17:10:05,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: writing mira0437250-226 from 2014508 to 2114409 tid 2 2014-07-01 17:10:05,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: read oid 440 snap -1 2014-07-01 17:10:05,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: expect (ObjNum 1360 snap 309 seq_num 1360) 2014-07-01 17:10:06,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 1 to mira0437250-226 2014-07-01 17:10:06,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 2 to mira0437250-226 2014-07-01 17:10:06,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 4 to mira0437250-226 2014-07-01 17:10:06,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: finishing write tid 5 to mira0437250-226 2014-07-01 17:10:06,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 806 (ObjNum 1426 snap 333 seq_num 1426) dirty exists 2014-07-01 17:10:06,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2980: done (2 left) 2014-07-01 17:10:06,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:2981: done (1 left) 2014-07-01 17:10:06,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: snap_create 2014-07-01 17:10:06,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: expect (ObjNum 695 snap 89 seq_num 695) 2014-07-01 17:10:06,673.673 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:06,673.673 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:06,673.673 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 17:10:06,674.674 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:10:06,744.744 INFO:teuthology.task.rados.rados.0.mira043.stdout:2982: done (1 left) 2014-07-01 17:10:06,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:2983: done (0 left) 2014-07-01 17:10:06,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:2984: read oid 262 snap -1 2014-07-01 17:10:06,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:2985: snap_create 2014-07-01 17:10:06,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:2984: expect (ObjNum 261 snap 0 seq_num 261) 2014-07-01 17:10:07,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 4 to mira0435601-230 2014-07-01 17:10:07,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3616: expect (ObjNum 1655 snap 0 seq_num 1655) 2014-07-01 17:10:07,433.433 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:10:07,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:2984: done (1 left) 2014-07-01 17:10:07,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2985: done (0 left) 2014-07-01 17:10:07,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: read oid 380 snap -1 2014-07-01 17:10:07,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: snap_create 2014-07-01 17:10:08,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:2986: done (1 left) 2014-07-01 17:10:08,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:2987: done (0 left) 2014-07-01 17:10:08,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: write oid 409 current snap is 336 2014-07-01 17:10:08,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: seq_num 1427 ranges {609639=714333,1809638=1} 2014-07-01 17:10:08,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: writing mira0437250-409 from 609639 to 1323972 tid 1 2014-07-01 17:10:08,762.762 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: writing mira0437250-409 from 1809638 to 1809639 tid 2 2014-07-01 17:10:08,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: rollback oid 488 current snap is 336 2014-07-01 17:10:08,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 488 to 315 2014-07-01 17:10:08,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:2990: snap_create 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 488 v 1100 (ObjNum 649 snap 69 seq_num 649) dirty exists 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 1 to mira0437250-409 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 2 to mira0437250-409 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 4 to mira0437250-409 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: finishing write tid 5 to mira0437250-409 2014-07-01 17:10:08,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 890 (ObjNum 1427 snap 336 seq_num 1427) dirty exists 2014-07-01 17:10:09,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:2988: done (2 left) 2014-07-01 17:10:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2989: done (1 left) 2014-07-01 17:10:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2990: done (0 left) 2014-07-01 17:10:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2991: read oid 392 snap -1 2014-07-01 17:10:09,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: read oid 394 snap -1 2014-07-01 17:10:09,743.743 INFO:teuthology.task.rados.rados.0.mira043.stdout:2993: snap_create 2014-07-01 17:10:09,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: expect (ObjNum 990 snap 188 seq_num 990) 2014-07-01 17:10:10,990.990 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:10:10,991.991 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 76' 2014-07-01 17:10:11,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 6 to mira0435601-230 2014-07-01 17:10:11,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: expect (ObjNum 2146 snap 0 seq_num 2146) 2014-07-01 17:10:11,784.784 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 76 2014-07-01 17:10:12,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:2991: done (2 left) 2014-07-01 17:10:12,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2992: done (1 left) 2014-07-01 17:10:12,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2993: done (0 left) 2014-07-01 17:10:12,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:2994: snap_create 2014-07-01 17:10:12,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2994: done (0 left) 2014-07-01 17:10:12,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: write oid 173 current snap is 339 2014-07-01 17:10:12,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: seq_num 1428 ranges {553444=437561,1406675=513077,2553443=1} 2014-07-01 17:10:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: writing mira0437250-173 from 553444 to 991005 tid 1 2014-07-01 17:10:12,830.830 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: writing mira0437250-173 from 1406675 to 1919752 tid 2 2014-07-01 17:10:12,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: writing mira0437250-173 from 2553443 to 2553444 tid 3 2014-07-01 17:10:12,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:2996: read oid 169 snap -1 2014-07-01 17:10:12,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: read oid 95 snap 338 2014-07-01 17:10:12,958.958 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: snap_create 2014-07-01 17:10:12,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: finishing write tid 1 to mira0437250-173 2014-07-01 17:10:12,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: finishing write tid 2 to mira0437250-173 2014-07-01 17:10:12,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: finishing write tid 3 to mira0437250-173 2014-07-01 17:10:12,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: finishing write tid 5 to mira0437250-173 2014-07-01 17:10:12,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:2996: expect (ObjNum 168 snap 0 seq_num 168) 2014-07-01 17:10:13,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: finishing write tid 6 to mira0437250-173 2014-07-01 17:10:13,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 1684 (ObjNum 1428 snap 339 seq_num 1428) dirty exists 2014-07-01 17:10:13,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: expect (ObjNum 1396 snap 319 seq_num 1396) 2014-07-01 17:10:15,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2995: done (3 left) 2014-07-01 17:10:15,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2996: done (2 left) 2014-07-01 17:10:15,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:2997: done (1 left) 2014-07-01 17:10:15,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2998: done (0 left) 2014-07-01 17:10:15,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:2999: read oid 8 snap -1 2014-07-01 17:10:15,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: read oid 367 snap -1 2014-07-01 17:10:15,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3001: snap_remove snap 312 2014-07-01 17:10:15,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: expect (ObjNum 985 snap 187 seq_num 985) 2014-07-01 17:10:16,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:2999: expect (ObjNum 7 snap 0 seq_num 7) 2014-07-01 17:10:16,800.800 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:16,800.800 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:16,801.801 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:10:16,801.801 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:10:16,801.801 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:10:16,802.802 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:10:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:2999: done (2 left) 2014-07-01 17:10:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3000: done (1 left) 2014-07-01 17:10:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3001: done (0 left) 2014-07-01 17:10:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: write oid 307 current snap is 340 2014-07-01 17:10:18,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: seq_num 1429 ranges {649698=409366,1520808=431848,2249697=1} 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0437250-307 from 649698 to 1059064 tid 1 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0437250-307 from 1520808 to 1952656 tid 2 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: writing mira0437250-307 from 2249697 to 2249698 tid 3 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3003: snap_remove snap 321 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 1 to mira0437250-307 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 2 to mira0437250-307 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 3 to mira0437250-307 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 5 to mira0437250-307 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: finishing write tid 6 to mira0437250-307 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 1483 (ObjNum 1429 snap 340 seq_num 1429) dirty exists 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: finishing write tid 1 to mira0435601-75 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: finishing write tid 2 to mira0435601-75 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: finishing write tid 4 to mira0435601-75 2014-07-01 17:10:18,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3612: expect (ObjNum 2004 snap 0 seq_num 2004) 2014-07-01 17:10:18,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:3002: done (1 left) 2014-07-01 17:10:18,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:3003: done (0 left) 2014-07-01 17:10:18,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: write oid 411 current snap is 340 2014-07-01 17:10:18,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: seq_num 1430 ranges {709856=417266,1599868=653467,2805236=678125,3509855=1} 2014-07-01 17:10:18,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0437250-411 from 709856 to 1127122 tid 1 2014-07-01 17:10:18,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0437250-411 from 1599868 to 2253335 tid 2 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0437250-411 from 2805236 to 3483361 tid 3 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: writing mira0437250-411 from 3509855 to 3509856 tid 4 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: read oid 471 snap -1 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: read oid 141 snap -1 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3007: snap_remove snap 317 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 1 to mira0437250-411 2014-07-01 17:10:19,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: expect (ObjNum 1264 snap 276 seq_num 1264) 2014-07-01 17:10:20,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: expect (ObjNum 1302 snap 295 seq_num 1302) 2014-07-01 17:10:21,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 2 to mira0437250-411 2014-07-01 17:10:21,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 3 to mira0437250-411 2014-07-01 17:10:21,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 4 to mira0437250-411 2014-07-01 17:10:21,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 6 to mira0437250-411 2014-07-01 17:10:21,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: finishing write tid 7 to mira0437250-411 2014-07-01 17:10:21,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 926 (ObjNum 1430 snap 340 seq_num 1430) dirty exists 2014-07-01 17:10:21,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3004: done (3 left) 2014-07-01 17:10:21,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3005: done (2 left) 2014-07-01 17:10:21,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3006: done (1 left) 2014-07-01 17:10:21,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3007: done (0 left) 2014-07-01 17:10:21,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3008: snap_create 2014-07-01 17:10:21,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3008: done (0 left) 2014-07-01 17:10:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: rollback oid 486 current snap is 341 2014-07-01 17:10:21,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 486 to 325 2014-07-01 17:10:21,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: read oid 331 snap -1 2014-07-01 17:10:21,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3011: snap_remove snap 299 2014-07-01 17:10:21,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: expect (ObjNum 1307 snap 297 seq_num 1307) 2014-07-01 17:10:22,267.267 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:22,267.267 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:22,267.267 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 17:10:22,267.267 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:10:22,267.267 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:10:22,268.268 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 17:10:23,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: finishing write tid 1 to mira0435601-476 2014-07-01 17:10:23,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: finishing write tid 3 to mira0435601-476 2014-07-01 17:10:23,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: finishing write tid 7 to mira0435601-230 2014-07-01 17:10:23,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 352 (ObjNum 2216 snap 0 seq_num 2216) dirty exists 2014-07-01 17:10:23,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: finishing write tid 5 to mira0435601-75 2014-07-01 17:10:23,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 291 (ObjNum 2217 snap 0 seq_num 2217) dirty exists 2014-07-01 17:10:23,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: finishing write tid 4 to mira0435601-476 2014-07-01 17:10:23,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 476 v 283 (ObjNum 2218 snap 0 seq_num 2218) dirty exists 2014-07-01 17:10:23,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: done (7 left) 2014-07-01 17:10:23,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: done (6 left) 2014-07-01 17:10:23,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: done (5 left) 2014-07-01 17:10:23,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3612: done (4 left) 2014-07-01 17:10:23,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: done (3 left) 2014-07-01 17:10:23,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: done (2 left) 2014-07-01 17:10:23,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: done (1 left) 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3616: done (0 left) 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3617: read oid 491 snap -1 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: seq_num 2219 ranges {3538944=622592} 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: writing mira0435601-228 from 3538944 to 4161536 tid 1 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 316 current snap is 0 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: seq_num 2220 ranges {5120000=491520} 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: writing mira0435601-316 from 5120000 to 5611520 tid 1 2014-07-01 17:10:23,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: read oid 35 snap -1 2014-07-01 17:10:23,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3617: expect (ObjNum 963 snap 0 seq_num 963) 2014-07-01 17:10:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 486 v 984 (ObjNum 1188 snap 251 seq_num 1188) dirty exists 2014-07-01 17:10:25,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:3009: done (2 left) 2014-07-01 17:10:25,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3010: done (1 left) 2014-07-01 17:10:25,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3011: done (0 left) 2014-07-01 17:10:25,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: write oid 454 current snap is 341 2014-07-01 17:10:25,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: seq_num 1431 ranges {254684=1} 2014-07-01 17:10:25,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: writing mira0437250-454 from 254684 to 254685 tid 1 2014-07-01 17:10:25,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:3013: rollback oid 138 current snap is 341 2014-07-01 17:10:25,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 138 to 334 2014-07-01 17:10:25,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: rollback oid 264 current snap is 341 2014-07-01 17:10:25,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 264 to 305 2014-07-01 17:10:25,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: write oid 157 current snap is 341 2014-07-01 17:10:25,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: seq_num 1432 ranges {749915=760875,2191012=427906,3205639=699258,3949914=1} 2014-07-01 17:10:25,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: writing mira0437250-157 from 749915 to 1510790 tid 1 2014-07-01 17:10:25,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: writing mira0437250-157 from 2191012 to 2618918 tid 2 2014-07-01 17:10:25,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: writing mira0437250-157 from 3205639 to 3904897 tid 3 2014-07-01 17:10:25,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: writing mira0437250-157 from 3949914 to 3949915 tid 4 2014-07-01 17:10:25,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3016: rollback oid 219 current snap is 341 2014-07-01 17:10:25,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 219 to 272 2014-07-01 17:10:25,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 138 v 791 (ObjNum 137 snap 0 seq_num 137) dirty exists 2014-07-01 17:10:25,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1506 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 17:10:25,020.020 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 1 to mira0437250-454 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 3 to mira0437250-454 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: finishing write tid 4 to mira0437250-454 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 1119 (ObjNum 1431 snap 341 seq_num 1431) dirty exists 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 1 to mira0437250-157 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 2 to mira0437250-157 2014-07-01 17:10:25,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3012: done (4 left) 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3013: done (3 left) 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3014: done (2 left) 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3017: snap_remove snap 335 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 3 to mira0437250-157 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 4 to mira0437250-157 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 6 to mira0437250-157 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: finishing write tid 7 to mira0437250-157 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 1512 (ObjNum 1432 snap 341 seq_num 1432) dirty exists 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3015: done (2 left) 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3017: done (1 left) 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: write oid 498 current snap is 341 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: seq_num 1433 ranges {694744=1} 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: writing mira0437250-498 from 694744 to 694745 tid 1 2014-07-01 17:10:25,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: write oid 242 current snap is 341 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: seq_num 1434 ranges {755927=740579,1955926=1} 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: writing mira0437250-242 from 755927 to 1496506 tid 1 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: writing mira0437250-242 from 1955926 to 1955927 tid 2 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: read oid 443 snap -1 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3021: snap_create 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 1 to mira0437250-498 2014-07-01 17:10:25,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 3 to mira0437250-498 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: finishing write tid 4 to mira0437250-498 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 859 (ObjNum 1433 snap 341 seq_num 1433) dirty exists 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: finishing write tid 1 to mira0437250-242 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: finishing write tid 2 to mira0437250-242 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: finishing write tid 4 to mira0437250-242 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: finishing write tid 5 to mira0437250-242 2014-07-01 17:10:25,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 922 (ObjNum 1434 snap 341 seq_num 1434) dirty exists 2014-07-01 17:10:25,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3617: done (3 left) 2014-07-01 17:10:25,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3621: read oid 211 snap -1 2014-07-01 17:10:25,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 17:10:25,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: seq_num 2221 ranges {4247552=520192,4767744=667648,5435392=536576} 2014-07-01 17:10:25,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: writing mira0435601-48 from 4247552 to 4767744 tid 1 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: writing mira0435601-48 from 4767744 to 5435392 tid 2 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: writing mira0435601-48 from 5435392 to 5971968 tid 3 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 137 current snap is 0 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: seq_num 2222 ranges {0=794624,794624=495616,1290240=307200} 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: writing mira0435601-137 from 0 to 794624 tid 1 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: writing mira0435601-137 from 794624 to 1290240 tid 2 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: writing mira0435601-137 from 1290240 to 1597440 tid 3 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 123 current snap is 0 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: seq_num 2223 ranges {3727360=712704,4440064=483328,4923392=495616,5419008=483328,5902336=143360} 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0435601-123 from 3727360 to 4440064 tid 1 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 1066 (ObjNum 1108 snap 222 seq_num 1108) dirty exists 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3016: done (4 left) 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3018: done (3 left) 2014-07-01 17:10:25,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3019: done (2 left) 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3020: done (1 left) 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3021: done (0 left) 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: rollback oid 482 current snap is 342 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 482 to 316 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: read oid 361 snap -1 2014-07-01 17:10:25,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3024: snap_remove snap 320 2014-07-01 17:10:25,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0435601-123 from 4440064 to 4923392 tid 2 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 482 v 1278 (ObjNum 971 snap 179 seq_num 971) dirty exists 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: expect (ObjNum 1404 snap 323 seq_num 1404) 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0435601-123 from 4923392 to 5419008 tid 3 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0435601-123 from 5419008 to 5902336 tid 4 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0435601-123 from 5902336 to 6045696 tid 5 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3625: read oid 149 snap -1 2014-07-01 17:10:26,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3621: expect (ObjNum 1698 snap 0 seq_num 1698) 2014-07-01 17:10:27,642.642 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 1 to mira0435601-228 2014-07-01 17:10:27,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: finishing write tid 1 to mira0435601-316 2014-07-01 17:10:27,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 3 to mira0435601-228 2014-07-01 17:10:27,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 1 to mira0435601-123 2014-07-01 17:10:27,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 2 to mira0435601-123 2014-07-01 17:10:27,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 3 to mira0435601-123 2014-07-01 17:10:27,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3621: done (7 left) 2014-07-01 17:10:27,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3626: read oid 138 snap -1 2014-07-01 17:10:27,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: finishing write tid 3 to mira0435601-316 2014-07-01 17:10:27,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: finishing write tid 1 to mira0435601-137 2014-07-01 17:10:27,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 4 to mira0435601-123 2014-07-01 17:10:27,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 5 to mira0435601-123 2014-07-01 17:10:27,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: finishing write tid 1 to mira0435601-48 2014-07-01 17:10:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: finishing write tid 2 to mira0435601-137 2014-07-01 17:10:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 7 to mira0435601-123 2014-07-01 17:10:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 8 to mira0435601-123 2014-07-01 17:10:27,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 260 (ObjNum 2223 snap 0 seq_num 2223) dirty exists 2014-07-01 17:10:27,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: finishing write tid 2 to mira0435601-48 2014-07-01 17:10:27,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: finishing write tid 3 to mira0435601-137 2014-07-01 17:10:27,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 4 to mira0435601-228 2014-07-01 17:10:27,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 173 (ObjNum 2219 snap 0 seq_num 2219) dirty exists 2014-07-01 17:10:27,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: finishing write tid 3 to mira0435601-48 2014-07-01 17:10:27,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: finishing write tid 5 to mira0435601-137 2014-07-01 17:10:27,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: finishing write tid 5 to mira0435601-48 2014-07-01 17:10:27,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: finishing write tid 4 to mira0435601-316 2014-07-01 17:10:27,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 231 (ObjNum 2220 snap 0 seq_num 2220) dirty exists 2014-07-01 17:10:27,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: finishing write tid 6 to mira0435601-137 2014-07-01 17:10:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 157 (ObjNum 2222 snap 0 seq_num 2222) dirty exists 2014-07-01 17:10:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: finishing write tid 6 to mira0435601-48 2014-07-01 17:10:27,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 398 (ObjNum 2221 snap 0 seq_num 2221) dirty exists 2014-07-01 17:10:27,741.741 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:27,741.741 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:27,741.741 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 17:10:27,741.741 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:10:27,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: expect (ObjNum 1803 snap 0 seq_num 1803) 2014-07-01 17:10:28,368.368 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:10:28,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3022: done (2 left) 2014-07-01 17:10:28,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:3023: done (1 left) 2014-07-01 17:10:28,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3024: done (0 left) 2014-07-01 17:10:28,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: rollback oid 311 current snap is 342 2014-07-01 17:10:28,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 311 to 336 2014-07-01 17:10:28,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: write oid 292 current snap is 342 2014-07-01 17:10:28,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: seq_num 1435 ranges {451156=712384,1651155=1} 2014-07-01 17:10:28,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0437250-292 from 451156 to 1163540 tid 1 2014-07-01 17:10:29,000.000 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: writing mira0437250-292 from 1651155 to 1651156 tid 2 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: snap_create 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 1501 (ObjNum 857 snap 143 seq_num 857) dirty exists 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 1 to mira0437250-292 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 2 to mira0437250-292 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 4 to mira0437250-292 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: finishing write tid 5 to mira0437250-292 2014-07-01 17:10:29,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 1524 (ObjNum 1435 snap 342 seq_num 1435) dirty exists 2014-07-01 17:10:29,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3025: done (2 left) 2014-07-01 17:10:29,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3026: done (1 left) 2014-07-01 17:10:29,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3027: done (0 left) 2014-07-01 17:10:29,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3028: snap_remove snap 319 2014-07-01 17:10:30,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3625: expect (ObjNum 1239 snap 0 seq_num 1239) 2014-07-01 17:10:30,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3028: done (0 left) 2014-07-01 17:10:30,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3029: rollback oid 198 current snap is 343 2014-07-01 17:10:30,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 198 to 279 2014-07-01 17:10:30,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3030: rollback oid 267 current snap is 343 2014-07-01 17:10:30,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 267 to 302 2014-07-01 17:10:30,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3031: rollback oid 381 current snap is 343 2014-07-01 17:10:30,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 381 to 316 2014-07-01 17:10:30,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: read oid 379 snap -1 2014-07-01 17:10:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: read oid 141 snap 291 2014-07-01 17:10:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3034: snap_create 2014-07-01 17:10:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 0 (ObjNum -1726349310 snap -2138581494 seq_num 0) dirty dne 2014-07-01 17:10:31,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 863 (ObjNum 938 snap 171 seq_num 938) dirty exists 2014-07-01 17:10:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 381 v 0 (ObjNum 30 snap 0 seq_num 1) dirty dne 2014-07-01 17:10:31,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: expect (ObjNum 1264 snap 276 seq_num 1264) 2014-07-01 17:10:32,112.112 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 76' 2014-07-01 17:10:32,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3029: done (5 left) 2014-07-01 17:10:32,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3030: done (4 left) 2014-07-01 17:10:32,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3031: done (3 left) 2014-07-01 17:10:32,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3032: done (2 left) 2014-07-01 17:10:32,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3033: done (1 left) 2014-07-01 17:10:32,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3034: done (0 left) 2014-07-01 17:10:32,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3035: snap_remove snap 341 2014-07-01 17:10:33,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3035: done (0 left) 2014-07-01 17:10:33,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: snap_remove snap 331 2014-07-01 17:10:33,080.080 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 76 2014-07-01 17:10:34,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3036: done (0 left) 2014-07-01 17:10:34,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: read oid 150 snap -1 2014-07-01 17:10:34,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3038: rollback oid 33 current snap is 344 2014-07-01 17:10:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 33 to 318 2014-07-01 17:10:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3039: snap_create 2014-07-01 17:10:34,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: done (7 left) 2014-07-01 17:10:34,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: done (6 left) 2014-07-01 17:10:34,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: done (5 left) 2014-07-01 17:10:34,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: done (4 left) 2014-07-01 17:10:34,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: done (3 left) 2014-07-01 17:10:34,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: done (2 left) 2014-07-01 17:10:34,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3625: done (1 left) 2014-07-01 17:10:34,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3626: done (0 left) 2014-07-01 17:10:34,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3627: delete oid 495 current snap is 0 2014-07-01 17:10:34,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 0 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:10:34,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: expect (ObjNum 1183 snap 250 seq_num 1183) 2014-07-01 17:10:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3037: done (2 left) 2014-07-01 17:10:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3038: done (1 left) 2014-07-01 17:10:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3039: done (0 left) 2014-07-01 17:10:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3040: rollback oid 146 current snap is 345 2014-07-01 17:10:35,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 146 to 344 2014-07-01 17:10:35,066.066 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: snap_remove snap 307 2014-07-01 17:10:35,074.074 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 146 v 0 (ObjNum 16555480 snap 0 seq_num 39427008) dirty dne 2014-07-01 17:10:36,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3040: done (1 left) 2014-07-01 17:10:36,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3041: done (0 left) 2014-07-01 17:10:36,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: write oid 362 current snap is 345 2014-07-01 17:10:36,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: seq_num 1436 ranges {643538=435612,1757173=460111,2640941=624148,3443537=1} 2014-07-01 17:10:36,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0437250-362 from 643538 to 1079150 tid 1 2014-07-01 17:10:36,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0437250-362 from 1757173 to 2217284 tid 2 2014-07-01 17:10:36,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0437250-362 from 2640941 to 3265089 tid 3 2014-07-01 17:10:36,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: writing mira0437250-362 from 3443537 to 3443538 tid 4 2014-07-01 17:10:36,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3043: snap_remove snap 318 2014-07-01 17:10:37,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:3043: done (1 left) 2014-07-01 17:10:37,077.077 INFO:teuthology.task.rados.rados.0.mira043.stdout:3044: delete oid 429 current snap is 345 2014-07-01 17:10:38,089.089 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:38,090.090 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:38,090.090 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:10:38,090.090 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:10:38,090.090 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:10:38,091.091 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:10:40,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 1 to mira0437250-362 2014-07-01 17:10:40,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 2 to mira0437250-362 2014-07-01 17:10:40,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 3 to mira0437250-362 2014-07-01 17:10:40,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 4 to mira0437250-362 2014-07-01 17:10:40,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 6 to mira0437250-362 2014-07-01 17:10:40,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: finishing write tid 7 to mira0437250-362 2014-07-01 17:10:40,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 362 v 1344 (ObjNum 1436 snap 345 seq_num 1436) dirty exists 2014-07-01 17:10:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3042: done (1 left) 2014-07-01 17:10:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3044: done (0 left) 2014-07-01 17:10:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: rollback oid 27 current snap is 345 2014-07-01 17:10:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 344 2014-07-01 17:10:40,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3046: snap_remove snap 332 2014-07-01 17:10:41,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3046: done (1 left) 2014-07-01 17:10:41,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3047: rollback oid 155 current snap is 345 2014-07-01 17:10:41,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 155 to 337 2014-07-01 17:10:41,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: snap_create 2014-07-01 17:10:42,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3048: done (2 left) 2014-07-01 17:10:42,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:3049: read oid 92 snap 328 2014-07-01 17:10:42,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: write oid 159 current snap is 346 2014-07-01 17:10:42,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: seq_num 1438 ranges {435022=415317,1336862=751925,2794816=40206} 2014-07-01 17:10:42,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0437250-159 from 435022 to 850339 tid 1 2014-07-01 17:10:42,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0437250-159 from 1336862 to 2088787 tid 2 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: writing mira0437250-159 from 2794816 to 2835022 tid 3 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: write oid 420 current snap is 346 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: seq_num 1439 ranges {744779=400000} 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: writing mira0437250-420 from 744779 to 1144779 tid 1 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: read oid 269 snap -1 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3049: done (5 left) 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3053: read oid 59 snap -1 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: rollback oid 99 current snap is 346 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 99 to 285 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: write oid 323 current snap is 346 2014-07-01 17:10:42,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: seq_num 1440 ranges {440009=400000} 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: writing mira0437250-323 from 440009 to 840009 tid 1 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: write oid 438 current snap is 346 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: seq_num 1441 ranges {536262=1} 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: writing mira0437250-438 from 536262 to 536263 tid 1 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3057: delete oid 69 current snap is 346 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 99 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:10:42,282.282 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: expect (ObjNum 268 snap 0 seq_num 268) 2014-07-01 17:10:43,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3627: done (0 left) 2014-07-01 17:10:43,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 480 current snap is 0 2014-07-01 17:10:43,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: seq_num 2225 ranges {69632=614400,684032=401408} 2014-07-01 17:10:43,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: writing mira0435601-480 from 69632 to 684032 tid 1 2014-07-01 17:10:43,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: writing mira0435601-480 from 684032 to 1085440 tid 2 2014-07-01 17:10:43,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3629: delete oid 60 current snap is 0 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 1 to mira0437250-438 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 3 to mira0437250-438 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: finishing write tid 4 to mira0437250-438 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 438 v 1015 (ObjNum 1441 snap 346 seq_num 1441) dirty exists 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: finishing write tid 1 to mira0437250-323 2014-07-01 17:10:43,415.415 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: finishing write tid 3 to mira0437250-323 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: finishing write tid 4 to mira0437250-323 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 1102 (ObjNum 1440 snap 346 seq_num 1440) dirty exists 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 0 (ObjNum 1 snap 1 seq_num 16568976) dirty dne 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 1478 (ObjNum 1296 snap 293 seq_num 1296) dirty exists 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: finishing write tid 1 to mira0437250-420 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: finishing write tid 3 to mira0437250-420 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: finishing write tid 4 to mira0437250-420 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 420 v 579 (ObjNum 1439 snap 346 seq_num 1439) dirty exists 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 1 to mira0437250-159 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 2 to mira0437250-159 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 3 to mira0437250-159 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 5 to mira0437250-159 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: finishing write tid 6 to mira0437250-159 2014-07-01 17:10:43,416.416 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 159 v 1485 (ObjNum 1438 snap 346 seq_num 1438) dirty exists 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3045: done (9 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3047: done (8 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3050: done (7 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3051: done (6 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3052: done (5 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3053: done (4 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3054: done (3 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3055: done (2 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3056: done (1 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3057: done (0 left) 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3058: snap_create 2014-07-01 17:10:43,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: finishing write tid 1 to mira0435601-480 2014-07-01 17:10:43,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: finishing write tid 2 to mira0435601-480 2014-07-01 17:10:43,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: finishing write tid 4 to mira0435601-480 2014-07-01 17:10:43,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3629: done (1 left) 2014-07-01 17:10:43,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 123 current snap is 0 2014-07-01 17:10:43,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: seq_num 2227 ranges {6045696=561152} 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: writing mira0435601-123 from 6045696 to 6606848 tid 1 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: read oid 73 snap -1 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: finishing write tid 5 to mira0435601-480 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 480 v 137 (ObjNum 2225 snap 0 seq_num 2225) dirty exists 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: done (2 left) 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 290 current snap is 0 2014-07-01 17:10:43,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: seq_num 2228 ranges {0=430080} 2014-07-01 17:10:43,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: writing mira0435601-290 from 0 to 430080 tid 1 2014-07-01 17:10:43,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:3633: read oid 71 snap -1 2014-07-01 17:10:43,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:3634: delete oid 281 current snap is 0 2014-07-01 17:10:43,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: finishing write tid 1 to mira0435601-123 2014-07-01 17:10:43,537.537 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:43,537.537 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:43,537.537 INFO:teuthology.task.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.547264 2014-07-01 17:10:43,537.537 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 3 0.547264095598' 2014-07-01 17:10:43,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3634: done (4 left) 2014-07-01 17:10:43,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3635: read oid 245 snap -1 2014-07-01 17:10:43,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3636: read oid 174 snap -1 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 232 current snap is 0 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: seq_num 2230 ranges {4161536=733184,4894720=614400,5509120=188416} 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: writing mira0435601-232 from 4161536 to 4894720 tid 1 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: writing mira0435601-232 from 4894720 to 5509120 tid 2 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: writing mira0435601-232 from 5509120 to 5697536 tid 3 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: read oid 12 snap -1 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: finishing write tid 3 to mira0435601-123 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: finishing write tid 4 to mira0435601-123 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 262 (ObjNum 2227 snap 0 seq_num 2227) dirty exists 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: done (7 left) 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3635: done (6 left) 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 172 current snap is 0 2014-07-01 17:10:43,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: seq_num 2231 ranges {9461760=602112,10063872=442368,10506240=360448} 2014-07-01 17:10:43,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0435601-172 from 9461760 to 10063872 tid 1 2014-07-01 17:10:43,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0435601-172 from 10063872 to 10506240 tid 2 2014-07-01 17:10:43,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0435601-172 from 10506240 to 10866688 tid 3 2014-07-01 17:10:43,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: read oid 158 snap -1 2014-07-01 17:10:43,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: finishing write tid 1 to mira0435601-232 2014-07-01 17:10:43,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: finishing write tid 2 to mira0435601-232 2014-07-01 17:10:43,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: finishing write tid 3 to mira0435601-232 2014-07-01 17:10:43,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: finishing write tid 5 to mira0435601-232 2014-07-01 17:10:43,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: finishing write tid 6 to mira0435601-232 2014-07-01 17:10:43,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 141 (ObjNum 2230 snap 0 seq_num 2230) dirty exists 2014-07-01 17:10:43,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: expect (ObjNum 1506 snap 0 seq_num 1506) 2014-07-01 17:10:44,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3058: done (0 left) 2014-07-01 17:10:44,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3059: rollback oid 245 current snap is 347 2014-07-01 17:10:44,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 245 to 323 2014-07-01 17:10:44,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3060: snap_remove snap 305 2014-07-01 17:10:44,179.179 INFO:teuthology.orchestra.run.mira079.stderr:set osd.3 primary-affinity to 0.547264 (8358652) 2014-07-01 17:10:45,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3060: done (1 left) 2014-07-01 17:10:45,139.139 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: delete oid 451 current snap is 347 2014-07-01 17:10:45,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3061: done (1 left) 2014-07-01 17:10:45,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: snap_create 2014-07-01 17:10:45,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 245 v 0 (ObjNum 1 snap 1 seq_num 16847280) dirty dne 2014-07-01 17:10:46,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3059: done (1 left) 2014-07-01 17:10:46,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3062: done (0 left) 2014-07-01 17:10:46,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: snap_remove snap 339 2014-07-01 17:10:46,924.924 INFO:teuthology.task.rados.rados.0.mira043.stdout:3636: expect (ObjNum 1655 snap 0 seq_num 1655) 2014-07-01 17:10:47,181.181 INFO:teuthology.task.rados.rados.0.mira043.stdout:3063: done (0 left) 2014-07-01 17:10:47,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: rollback oid 82 current snap is 348 2014-07-01 17:10:47,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 82 to 325 2014-07-01 17:10:47,182.182 INFO:teuthology.task.rados.rados.0.mira043.stdout:3065: read oid 122 snap -1 2014-07-01 17:10:47,183.183 INFO:teuthology.task.rados.rados.0.mira043.stdout:3066: snap_remove snap 342 2014-07-01 17:10:47,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 0 (ObjNum 26860832 snap 0 seq_num 32) dirty dne 2014-07-01 17:10:48,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3064: done (2 left) 2014-07-01 17:10:48,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3065: done (1 left) 2014-07-01 17:10:48,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3066: done (0 left) 2014-07-01 17:10:48,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3067: snap_remove snap 328 2014-07-01 17:10:49,192.192 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:49,195.195 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:49,195.195 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 1 2014-07-01 17:10:49,195.195 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:10:49,196.196 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:10:49,196.196 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok config set filestore_inject_stall 3' 2014-07-01 17:10:49,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3067: done (0 left) 2014-07-01 17:10:49,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: read oid 207 snap -1 2014-07-01 17:10:49,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: snap_create 2014-07-01 17:10:50,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3068: done (1 left) 2014-07-01 17:10:50,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3069: done (0 left) 2014-07-01 17:10:50,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: read oid 454 snap -1 2014-07-01 17:10:50,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: read oid 106 snap -1 2014-07-01 17:10:50,242.242 INFO:teuthology.task.rados.rados.0.mira043.stdout:3072: delete oid 137 current snap is 349 2014-07-01 17:10:50,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: expect (ObjNum 1431 snap 341 seq_num 1431) 2014-07-01 17:10:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3070: done (2 left) 2014-07-01 17:10:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3072: done (1 left) 2014-07-01 17:10:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: delete oid 382 current snap is 349 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: expect (ObjNum 863 snap 145 seq_num 863) 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 1 to mira0435601-172 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 2 to mira0435601-172 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: finishing write tid 1 to mira0435601-290 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: finishing write tid 3 to mira0435601-290 2014-07-01 17:10:50,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: expect (ObjNum 1975 snap 0 seq_num 1975) 2014-07-01 17:10:51,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3071: done (1 left) 2014-07-01 17:10:51,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3073: done (0 left) 2014-07-01 17:10:51,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: write oid 191 current snap is 349 2014-07-01 17:10:51,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: seq_num 1446 ranges {525116=413368,1687359=699982,2979722=574293,3725115=1} 2014-07-01 17:10:51,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: writing mira0437250-191 from 525116 to 938484 tid 1 2014-07-01 17:10:51,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: writing mira0437250-191 from 1687359 to 2387341 tid 2 2014-07-01 17:10:51,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: writing mira0437250-191 from 2979722 to 3554015 tid 3 2014-07-01 17:10:51,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: writing mira0437250-191 from 3725115 to 3725116 tid 4 2014-07-01 17:10:51,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: write oid 125 current snap is 349 2014-07-01 17:10:51,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: seq_num 1447 ranges {620345=785172,1916818=521602,3047699=372646} 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: writing mira0437250-125 from 620345 to 1405517 tid 1 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: writing mira0437250-125 from 1916818 to 2438420 tid 2 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: writing mira0437250-125 from 3047699 to 3420345 tid 3 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: delete oid 364 current snap is 349 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 1 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 2 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 3 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 4 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 6 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: finishing write tid 7 to mira0437250-191 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 1057 (ObjNum 1446 snap 349 seq_num 1446) dirty exists 2014-07-01 17:10:51,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 1 to mira0437250-125 2014-07-01 17:10:51,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 2 to mira0437250-125 2014-07-01 17:10:51,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:3074: done (2 left) 2014-07-01 17:10:51,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:3076: done (1 left) 2014-07-01 17:10:51,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: write oid 202 current snap is 349 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: seq_num 1449 ranges {626357=481229,1426356=1} 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: writing mira0437250-202 from 626357 to 1107586 tid 1 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: writing mira0437250-202 from 1426356 to 1426357 tid 2 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: read oid 211 snap -1 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: read oid 105 snap -1 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 3 to mira0437250-125 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 5 to mira0437250-125 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: finishing write tid 6 to mira0437250-125 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 125 v 758 (ObjNum 1447 snap 349 seq_num 1447) dirty exists 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3075: done (3 left) 2014-07-01 17:10:51,259.259 INFO:teuthology.task.rados.rados.0.mira043.stdout:3080: rollback oid 54 current snap is 349 2014-07-01 17:10:51,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 54 to 303 2014-07-01 17:10:51,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: read oid 142 snap -1 2014-07-01 17:10:51,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: rollback oid 363 current snap is 349 2014-07-01 17:10:51,260.260 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 363 to 272 2014-07-01 17:10:51,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3083: rollback oid 316 current snap is 349 2014-07-01 17:10:51,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 316 to 278 2014-07-01 17:10:51,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: write oid 191 current snap is 349 2014-07-01 17:10:51,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: seq_num 1450 ranges {721587=400000} 2014-07-01 17:10:51,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: writing mira0437250-191 from 721587 to 1121587 tid 1 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: read oid 56 snap -1 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 0 (ObjNum 16994976 snap 0 seq_num 16503920) dirty dne 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 54 v 832 (ObjNum 53 snap 0 seq_num 53) dirty exists 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 363 v 1316 (ObjNum 362 snap 0 seq_num 362) dirty exists 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3078: done (8 left) 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3079: done (7 left) 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3080: done (6 left) 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3082: done (5 left) 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3083: done (4 left) 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: write oid 402 current snap is 349 2014-07-01 17:10:51,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: seq_num 1451 ranges {666416=460933,1635426=230990} 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: writing mira0437250-402 from 666416 to 1127349 tid 1 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: writing mira0437250-402 from 1635426 to 1866416 tid 2 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3087: snap_create 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: finishing write tid 1 to mira0437250-202 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: finishing write tid 2 to mira0437250-202 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: finishing write tid 4 to mira0437250-202 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: finishing write tid 5 to mira0437250-202 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 202 v 960 (ObjNum 1449 snap 349 seq_num 1449) dirty exists 2014-07-01 17:10:51,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 17:10:51,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 1 to mira0437250-191 2014-07-01 17:10:51,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 3 to mira0437250-191 2014-07-01 17:10:51,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: finishing write tid 4 to mira0437250-191 2014-07-01 17:10:51,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 191 v 1059 (ObjNum 1450 snap 349 seq_num 1450) dirty exists 2014-07-01 17:10:51,541.541 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: finishing write tid 1 to mira0437250-402 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: finishing write tid 2 to mira0437250-402 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: finishing write tid 4 to mira0437250-402 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: finishing write tid 5 to mira0437250-402 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 1056 (ObjNum 1451 snap 349 seq_num 1451) dirty exists 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3077: done (5 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3081: done (4 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3084: done (3 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3085: done (2 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3086: done (1 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3087: done (0 left) 2014-07-01 17:10:51,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3088: read oid 62 snap 316 2014-07-01 17:10:51,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:3089: delete oid 359 current snap is 350 2014-07-01 17:10:51,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:3089: done (1 left) 2014-07-01 17:10:51,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: write oid 131 current snap is 350 2014-07-01 17:10:51,551.551 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: seq_num 1453 ranges {421804=556990,1543948=576821,2821803=1} 2014-07-01 17:10:51,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: writing mira0437250-131 from 421804 to 978794 tid 1 2014-07-01 17:10:51,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: writing mira0437250-131 from 1543948 to 2120769 tid 2 2014-07-01 17:10:51,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: writing mira0437250-131 from 2821803 to 2821804 tid 3 2014-07-01 17:10:51,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3091: snap_create 2014-07-01 17:10:51,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: finishing write tid 1 to mira0437250-131 2014-07-01 17:10:51,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: finishing write tid 2 to mira0437250-131 2014-07-01 17:10:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: finishing write tid 3 to mira0437250-131 2014-07-01 17:10:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: finishing write tid 5 to mira0437250-131 2014-07-01 17:10:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: finishing write tid 6 to mira0437250-131 2014-07-01 17:10:51,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 980 (ObjNum 1453 snap 350 seq_num 1453) dirty exists 2014-07-01 17:10:52,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3088: done (2 left) 2014-07-01 17:10:52,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3090: done (1 left) 2014-07-01 17:10:52,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3091: done (0 left) 2014-07-01 17:10:52,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3092: snap_remove snap 334 2014-07-01 17:10:53,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3092: done (0 left) 2014-07-01 17:10:53,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: snap_create 2014-07-01 17:10:53,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 3 to mira0435601-172 2014-07-01 17:10:53,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 5 to mira0435601-172 2014-07-01 17:10:53,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 6 to mira0435601-172 2014-07-01 17:10:53,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 268 (ObjNum 2231 snap 0 seq_num 2231) dirty exists 2014-07-01 17:10:53,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: expect (ObjNum 1980 snap 0 seq_num 1980) 2014-07-01 17:10:54,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:3093: done (0 left) 2014-07-01 17:10:54,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: read oid 274 snap -1 2014-07-01 17:10:54,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: read oid 300 snap -1 2014-07-01 17:10:54,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: snap_remove snap 351 2014-07-01 17:10:54,411.411 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: expect (ObjNum 635 snap 60 seq_num 635) 2014-07-01 17:10:54,753.753 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:10:54,754.754 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:10:54,754.754 INFO:teuthology.task.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2014-07-01 17:10:54,754.754 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 0 0' 2014-07-01 17:10:54,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: expect (ObjNum 1173 snap 246 seq_num 1173) 2014-07-01 17:10:55,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3094: done (2 left) 2014-07-01 17:10:55,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3095: done (1 left) 2014-07-01 17:10:55,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3096: done (0 left) 2014-07-01 17:10:55,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: write oid 220 current snap is 352 2014-07-01 17:10:55,612.612 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: seq_num 1454 ranges {766633=412442,1790303=798440,3067477=499156} 2014-07-01 17:10:55,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0437250-220 from 766633 to 1179075 tid 1 2014-07-01 17:10:55,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0437250-220 from 1790303 to 2588743 tid 2 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: writing mira0437250-220 from 3067477 to 3566633 tid 3 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: read oid 167 snap -1 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 1 to mira0437250-220 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: write oid 259 current snap is 352 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: seq_num 1455 ranges {461863=785271,1868340=620059,2984031=277832} 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: writing mira0437250-259 from 461863 to 1247134 tid 1 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: writing mira0437250-259 from 1868340 to 2488399 tid 2 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: writing mira0437250-259 from 2984031 to 3261863 tid 3 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3100: rollback oid 264 current snap is 352 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 264 to 348 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 2 to mira0437250-220 2014-07-01 17:10:55,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: expect (ObjNum 166 snap 0 seq_num 166) 2014-07-01 17:10:55,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: finishing write tid 4 to mira0435601-290 2014-07-01 17:10:55,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 228 (ObjNum 2228 snap 0 seq_num 2228) dirty exists 2014-07-01 17:10:55,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:3633: expect (ObjNum 1892 snap 0 seq_num 1892) 2014-07-01 17:10:55,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 3 to mira0437250-220 2014-07-01 17:10:55,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 5 to mira0437250-220 2014-07-01 17:10:55,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: finishing write tid 6 to mira0437250-220 2014-07-01 17:10:55,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 987 (ObjNum 1454 snap 352 seq_num 1454) dirty exists 2014-07-01 17:10:55,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: finishing write tid 1 to mira0437250-259 2014-07-01 17:10:55,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: finishing write tid 2 to mira0437250-259 2014-07-01 17:10:55,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1584 (ObjNum 263 snap 0 seq_num 263) dirty exists 2014-07-01 17:10:55,990.990 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: finishing write tid 3 to mira0437250-259 2014-07-01 17:10:55,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: finishing write tid 5 to mira0437250-259 2014-07-01 17:10:55,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: finishing write tid 6 to mira0437250-259 2014-07-01 17:10:55,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 259 v 1169 (ObjNum 1455 snap 352 seq_num 1455) dirty exists 2014-07-01 17:10:55,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3097: done (3 left) 2014-07-01 17:10:55,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3098: done (2 left) 2014-07-01 17:10:55,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3099: done (1 left) 2014-07-01 17:10:55,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3100: done (0 left) 2014-07-01 17:10:55,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: delete oid 9 current snap is 352 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3101: done (0 left) 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: read oid 390 snap 325 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: rollback oid 15 current snap is 352 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 15 to 347 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: snap_create 2014-07-01 17:10:56,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 1192 (ObjNum 1038 snap 205 seq_num 1038) dirty exists 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3102: done (2 left) 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3103: done (1 left) 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3104: done (0 left) 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3105: read oid 171 snap -1 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: write oid 493 current snap is 353 2014-07-01 17:10:56,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: seq_num 1457 ranges {466851=480303,1266850=1} 2014-07-01 17:10:56,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: writing mira0437250-493 from 466851 to 947154 tid 1 2014-07-01 17:10:56,307.307 INFO:teuthology.orchestra.run.mira079.stderr:set osd.0 primary-affinity to 0 (802) 2014-07-01 17:10:56,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: writing mira0437250-493 from 1266850 to 1266851 tid 2 2014-07-01 17:10:56,435.435 INFO:teuthology.task.rados.rados.0.mira043.stdout:3107: read oid 268 snap -1 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3108: delete oid 377 current snap is 353 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: finishing write tid 1 to mira0437250-493 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: finishing write tid 2 to mira0437250-493 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: finishing write tid 4 to mira0437250-493 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3105: done (3 left) 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3107: done (2 left) 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3108: done (1 left) 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: delete oid 68 current snap is 353 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: finishing write tid 5 to mira0437250-493 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 634 (ObjNum 1457 snap 353 seq_num 1457) dirty exists 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3106: done (1 left) 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3109: done (0 left) 2014-07-01 17:10:56,436.436 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: snap_remove snap 316 2014-07-01 17:10:57,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3110: done (0 left) 2014-07-01 17:10:57,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: snap_create 2014-07-01 17:10:58,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3111: done (0 left) 2014-07-01 17:10:58,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: write oid 295 current snap is 354 2014-07-01 17:10:58,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: seq_num 1460 ranges {450716=583236,1566808=606107,2450715=1} 2014-07-01 17:10:58,320.320 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: writing mira0437250-295 from 450716 to 1033952 tid 1 2014-07-01 17:10:58,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: writing mira0437250-295 from 1566808 to 2172915 tid 2 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: writing mira0437250-295 from 2450715 to 2450716 tid 3 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: read oid 161 snap 302 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: finishing write tid 1 to mira0437250-295 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3114: rollback oid 495 current snap is 354 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 495 to 314 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: read oid 44 snap -1 2014-07-01 17:10:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3116: rollback oid 386 current snap is 354 2014-07-01 17:10:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 386 to 285 2014-07-01 17:10:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:3117: snap_create 2014-07-01 17:10:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 1062 (ObjNum 827 snap 132 seq_num 827) dirty exists 2014-07-01 17:10:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 386 v 998 (ObjNum 947 snap 173 seq_num 947) dirty exists 2014-07-01 17:10:58,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: expect (ObjNum 844 snap 141 seq_num 844) 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: finishing write tid 2 to mira0437250-295 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: finishing write tid 3 to mira0437250-295 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: finishing write tid 5 to mira0437250-295 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: finishing write tid 6 to mira0437250-295 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 1521 (ObjNum 1460 snap 354 seq_num 1460) dirty exists 2014-07-01 17:10:58,829.829 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: expect (ObjNum 1225 snap 264 seq_num 1225) 2014-07-01 17:11:00,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3112: done (5 left) 2014-07-01 17:11:00,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3113: done (4 left) 2014-07-01 17:11:00,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3114: done (3 left) 2014-07-01 17:11:00,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3115: done (2 left) 2014-07-01 17:11:00,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3116: done (1 left) 2014-07-01 17:11:00,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3117: done (0 left) 2014-07-01 17:11:00,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: snap_remove snap 302 2014-07-01 17:11:00,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3118: done (0 left) 2014-07-01 17:11:00,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: read oid 132 snap 325 2014-07-01 17:11:00,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: write oid 91 current snap is 355 2014-07-01 17:11:00,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: seq_num 1461 ranges {663322=555041,1645869=427726,2663321=1} 2014-07-01 17:11:00,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0437250-91 from 663322 to 1218363 tid 1 2014-07-01 17:11:00,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0437250-91 from 1645869 to 2073595 tid 2 2014-07-01 17:11:00,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: writing mira0437250-91 from 2663321 to 2663322 tid 3 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: delete oid 302 current snap is 355 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3119: done (2 left) 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3121: done (1 left) 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: delete oid 206 current snap is 355 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3122: done (1 left) 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: read oid 257 snap 315 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3124: read oid 353 snap -1 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3125: snap_remove snap 278 2014-07-01 17:11:00,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: expect (ObjNum 256 snap 0 seq_num 256) 2014-07-01 17:11:00,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: done (7 left) 2014-07-01 17:11:00,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: done (6 left) 2014-07-01 17:11:00,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3633: done (5 left) 2014-07-01 17:11:00,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3636: done (4 left) 2014-07-01 17:11:00,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: done (3 left) 2014-07-01 17:11:00,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: done (2 left) 2014-07-01 17:11:00,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: done (1 left) 2014-07-01 17:11:00,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: done (0 left) 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3641: read oid 386 snap -1 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 405 current snap is 0 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: seq_num 2232 ranges {0=679936,679936=401408} 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: writing mira0435601-405 from 0 to 679936 tid 1 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: writing mira0435601-405 from 679936 to 1081344 tid 2 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3643: read oid 304 snap -1 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3641: expect (ObjNum 1246 snap 0 seq_num 1246) 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: finishing write tid 1 to mira0435601-405 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3643: expect (ObjNum 1300 snap 0 seq_num 1300) 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 1 to mira0437250-91 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 2 to mira0437250-91 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 3 to mira0437250-91 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 5 to mira0437250-91 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: finishing write tid 6 to mira0437250-91 2014-07-01 17:11:00,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 91 v 1456 (ObjNum 1461 snap 355 seq_num 1461) dirty exists 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: finishing write tid 2 to mira0435601-405 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: finishing write tid 4 to mira0435601-405 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: finishing write tid 5 to mira0435601-405 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 294 (ObjNum 2232 snap 0 seq_num 2232) dirty exists 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3641: done (2 left) 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: done (1 left) 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3643: done (0 left) 2014-07-01 17:11:00,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 33 current snap is 0 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: seq_num 2233 ranges {1183744=552960,1736704=401408} 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: writing mira0435601-33 from 1183744 to 1736704 tid 1 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: writing mira0435601-33 from 1736704 to 2138112 tid 2 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3645: read oid 359 snap -1 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3646: read oid 174 snap -1 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3647: read oid 210 snap -1 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 344 current snap is 0 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: seq_num 2234 ranges {1527808=421888,1949696=401408} 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: writing mira0435601-344 from 1527808 to 1949696 tid 1 2014-07-01 17:11:00,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: writing mira0435601-344 from 1949696 to 2351104 tid 2 2014-07-01 17:11:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3649: read oid 103 snap -1 2014-07-01 17:11:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: finishing write tid 1 to mira0435601-33 2014-07-01 17:11:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: finishing write tid 2 to mira0435601-33 2014-07-01 17:11:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: finishing write tid 4 to mira0435601-33 2014-07-01 17:11:00,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: finishing write tid 5 to mira0435601-33 2014-07-01 17:11:00,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 355 (ObjNum 2233 snap 0 seq_num 2233) dirty exists 2014-07-01 17:11:00,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: done (5 left) 2014-07-01 17:11:00,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3647: done (4 left) 2014-07-01 17:11:00,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: read oid 393 snap -1 2014-07-01 17:11:00,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3651: delete oid 487 current snap is 0 2014-07-01 17:11:00,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: finishing write tid 1 to mira0435601-344 2014-07-01 17:11:00,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: finishing write tid 2 to mira0435601-344 2014-07-01 17:11:00,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: finishing write tid 4 to mira0435601-344 2014-07-01 17:11:00,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3651: done (5 left) 2014-07-01 17:11:00,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3652: read oid 239 snap -1 2014-07-01 17:11:00,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 366 current snap is 0 2014-07-01 17:11:00,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: seq_num 2236 ranges {2457600=372736} 2014-07-01 17:11:00,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: writing mira0435601-366 from 2457600 to 2830336 tid 1 2014-07-01 17:11:01,012.012 INFO:teuthology.task.rados.rados.0.mira043.stdout:3654: delete oid 172 current snap is 0 2014-07-01 17:11:01,013.013 INFO:teuthology.task.rados.rados.0.mira043.stdout:3646: expect (ObjNum 1655 snap 0 seq_num 1655) 2014-07-01 17:11:01,316.316 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:01,316.316 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:01,316.316 INFO:teuthology.task.thrashosds.thrasher:Reweighting osd 0 to 0.220952543531 2014-07-01 17:11:01,317.317 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 0.220952543531' 2014-07-01 17:11:01,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3120: done (3 left) 2014-07-01 17:11:01,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3123: done (2 left) 2014-07-01 17:11:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3124: done (1 left) 2014-07-01 17:11:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3125: done (0 left) 2014-07-01 17:11:01,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3126: snap_remove snap 337 2014-07-01 17:11:02,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:3126: done (0 left) 2014-07-01 17:11:02,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: write oid 325 current snap is 355 2014-07-01 17:11:02,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: seq_num 1464 ranges {648211=506550,1684393=638312,2873011=623819,3848210=1} 2014-07-01 17:11:02,375.375 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 0.220953 (8144802) 2014-07-01 17:11:02,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: writing mira0437250-325 from 648211 to 1154761 tid 1 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: writing mira0437250-325 from 1684393 to 2322705 tid 2 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: writing mira0437250-325 from 2873011 to 3496830 tid 3 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: writing mira0437250-325 from 3848210 to 3848211 tid 4 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: write oid 46 current snap is 355 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: seq_num 1465 ranges {743441=478354,1913853=459931,2790589=509585,3543440=1} 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0437250-46 from 743441 to 1221795 tid 1 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0437250-46 from 1913853 to 2373784 tid 2 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0437250-46 from 2790589 to 3300174 tid 3 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: writing mira0437250-46 from 3543440 to 3543441 tid 4 2014-07-01 17:11:02,535.535 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: snap_remove snap 350 2014-07-01 17:11:02,544.544 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 1 to mira0437250-325 2014-07-01 17:11:02,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 2 to mira0437250-325 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 3 to mira0437250-325 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 4 to mira0437250-325 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 6 to mira0437250-325 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: finishing write tid 7 to mira0437250-325 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 325 v 1110 (ObjNum 1464 snap 355 seq_num 1464) dirty exists 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 1 to mira0437250-46 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 2 to mira0437250-46 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 3 to mira0437250-46 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 4 to mira0437250-46 2014-07-01 17:11:02,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 6 to mira0437250-46 2014-07-01 17:11:02,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: finishing write tid 7 to mira0437250-46 2014-07-01 17:11:02,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 46 v 834 (ObjNum 1465 snap 355 seq_num 1465) dirty exists 2014-07-01 17:11:03,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:3127: done (2 left) 2014-07-01 17:11:03,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:3128: done (1 left) 2014-07-01 17:11:03,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:3129: done (0 left) 2014-07-01 17:11:03,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:3130: read oid 376 snap 323 2014-07-01 17:11:03,383.383 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: write oid 360 current snap is 355 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: seq_num 1466 ranges {653198=486254,1853197=1} 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0437250-360 from 653198 to 1139452 tid 1 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: writing mira0437250-360 from 1853197 to 1853198 tid 2 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: read oid 210 snap -1 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: write oid 407 current snap is 355 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: seq_num 1467 ranges {748428=458059,1548427=1} 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: writing mira0437250-407 from 748428 to 1206487 tid 1 2014-07-01 17:11:03,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: writing mira0437250-407 from 1548427 to 1548428 tid 2 2014-07-01 17:11:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3134: snap_create 2014-07-01 17:11:03,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: expect (ObjNum 987 snap 187 seq_num 987) 2014-07-01 17:11:04,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: finishing write tid 1 to mira0437250-407 2014-07-01 17:11:04,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 1 to mira0437250-360 2014-07-01 17:11:04,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: finishing write tid 2 to mira0437250-407 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: finishing write tid 4 to mira0437250-407 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: finishing write tid 5 to mira0437250-407 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 407 v 1042 (ObjNum 1467 snap 355 seq_num 1467) dirty exists 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 2 to mira0437250-360 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 4 to mira0437250-360 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: finishing write tid 5 to mira0437250-360 2014-07-01 17:11:04,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 964 (ObjNum 1466 snap 355 seq_num 1466) dirty exists 2014-07-01 17:11:04,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:3130: done (4 left) 2014-07-01 17:11:04,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3131: done (3 left) 2014-07-01 17:11:04,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3132: done (2 left) 2014-07-01 17:11:04,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3133: done (1 left) 2014-07-01 17:11:04,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3134: done (0 left) 2014-07-01 17:11:04,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3135: snap_remove snap 313 2014-07-01 17:11:04,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: expect (ObjNum 2186 snap 0 seq_num 2186) 2014-07-01 17:11:05,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3135: done (0 left) 2014-07-01 17:11:05,402.402 INFO:teuthology.task.rados.rados.0.mira043.stdout:3136: snap_create 2014-07-01 17:11:06,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: finishing write tid 5 to mira0435601-344 2014-07-01 17:11:06,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 382 (ObjNum 2234 snap 0 seq_num 2234) dirty exists 2014-07-01 17:11:06,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3645: expect (ObjNum 1542 snap 0 seq_num 1542) 2014-07-01 17:11:07,397.397 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:07,398.398 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:07,398.398 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 4 2014-07-01 17:11:07,399.399 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:11:07,399.399 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:11:07,399.399 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.4.asok config set filestore_inject_stall 3' 2014-07-01 17:11:08,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3136: done (0 left) 2014-07-01 17:11:08,280.280 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: read oid 138 snap -1 2014-07-01 17:11:08,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: write oid 244 current snap is 357 2014-07-01 17:11:08,281.281 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: seq_num 1468 ranges {443658=581287,1243657=1} 2014-07-01 17:11:08,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: writing mira0437250-244 from 443658 to 1024945 tid 1 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: writing mira0437250-244 from 1243657 to 1243658 tid 2 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: read oid 175 snap -1 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: rollback oid 305 current snap is 357 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 305 to 333 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: snap_create 2014-07-01 17:11:08,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 17:11:08,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 1 to mira0437250-244 2014-07-01 17:11:08,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 2 to mira0437250-244 2014-07-01 17:11:08,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 4 to mira0437250-244 2014-07-01 17:11:08,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: finishing write tid 5 to mira0437250-244 2014-07-01 17:11:08,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 244 v 1262 (ObjNum 1468 snap 357 seq_num 1468) dirty exists 2014-07-01 17:11:08,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 1255 (ObjNum 1234 snap 269 seq_num 1234) dirty exists 2014-07-01 17:11:09,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3138: done (4 left) 2014-07-01 17:11:09,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3139: done (3 left) 2014-07-01 17:11:09,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3140: done (2 left) 2014-07-01 17:11:09,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:3141: done (1 left) 2014-07-01 17:11:09,312.312 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: write oid 58 current snap is 358 2014-07-01 17:11:09,473.473 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: seq_num 1469 ranges {637064=554115,1632462=527208,2929763=107301} 2014-07-01 17:11:09,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0437250-58 from 637064 to 1191179 tid 1 2014-07-01 17:11:09,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0437250-58 from 1632462 to 2159670 tid 2 2014-07-01 17:11:09,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: writing mira0437250-58 from 2929763 to 3037064 tid 3 2014-07-01 17:11:09,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3143: delete oid 51 current snap is 358 2014-07-01 17:11:10,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3652: expect (ObjNum 1620 snap 0 seq_num 1620) 2014-07-01 17:11:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: finishing write tid 1 to mira0435601-366 2014-07-01 17:11:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: finishing write tid 3 to mira0435601-366 2014-07-01 17:11:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: finishing write tid 4 to mira0435601-366 2014-07-01 17:11:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 240 (ObjNum 2236 snap 0 seq_num 2236) dirty exists 2014-07-01 17:11:10,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3649: expect (ObjNum 1643 snap 0 seq_num 1643) 2014-07-01 17:11:11,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3645: done (7 left) 2014-07-01 17:11:11,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3646: done (6 left) 2014-07-01 17:11:11,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: done (5 left) 2014-07-01 17:11:11,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3649: done (4 left) 2014-07-01 17:11:11,690.690 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: done (3 left) 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3652: done (2 left) 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: done (1 left) 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3654: done (0 left) 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 215 current snap is 0 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: seq_num 2238 ranges {5955584=671744,6627328=421888,7049216=380928} 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: writing mira0435601-215 from 5955584 to 6627328 tid 1 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: writing mira0435601-215 from 6627328 to 7049216 tid 2 2014-07-01 17:11:11,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: writing mira0435601-215 from 7049216 to 7430144 tid 3 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: read oid 65 snap -1 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 337 current snap is 0 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: seq_num 2239 ranges {0=544768,544768=651264,1196032=151552} 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: writing mira0435601-337 from 0 to 544768 tid 1 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: writing mira0435601-337 from 544768 to 1196032 tid 2 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: writing mira0435601-337 from 1196032 to 1347584 tid 3 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: read oid 436 snap -1 2014-07-01 17:11:11,854.854 INFO:teuthology.task.rados.rados.0.mira043.stdout:3659: delete oid 378 current snap is 0 2014-07-01 17:11:11,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: expect (ObjNum 1653 snap 0 seq_num 1653) 2014-07-01 17:11:12,848.848 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:12,849.849 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:12,849.849 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 17:11:12,849.849 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:11:12,850.850 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:11:12,850.850 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 17:11:13,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 1 to mira0437250-58 2014-07-01 17:11:13,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 2 to mira0437250-58 2014-07-01 17:11:13,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 3 to mira0437250-58 2014-07-01 17:11:13,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 5 to mira0437250-58 2014-07-01 17:11:13,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:3143: done (2 left) 2014-07-01 17:11:13,597.597 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: snap_remove snap 323 2014-07-01 17:11:15,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:3144: done (2 left) 2014-07-01 17:11:15,147.147 INFO:teuthology.task.rados.rados.0.mira043.stdout:3145: snap_remove snap 343 2014-07-01 17:11:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: finishing write tid 1 to mira0435601-215 2014-07-01 17:11:16,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: finishing write tid 2 to mira0435601-215 2014-07-01 17:11:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: finishing write tid 3 to mira0435601-215 2014-07-01 17:11:16,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: finishing write tid 5 to mira0435601-215 2014-07-01 17:11:16,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: finishing write tid 1 to mira0435601-337 2014-07-01 17:11:16,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: finishing write tid 2 to mira0435601-337 2014-07-01 17:11:16,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: finishing write tid 3 to mira0435601-337 2014-07-01 17:11:16,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: finishing write tid 5 to mira0435601-337 2014-07-01 17:11:16,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: finishing write tid 6 to mira0435601-337 2014-07-01 17:11:16,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 337 v 386 (ObjNum 2239 snap 0 seq_num 2239) dirty exists 2014-07-01 17:11:16,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3145: done (2 left) 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: write oid 477 current snap is 358 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: seq_num 1471 ranges {544899=532796,1575030=667598,2778741=166158} 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: writing mira0437250-477 from 544899 to 1077695 tid 1 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: writing mira0437250-477 from 1575030 to 2242628 tid 2 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: writing mira0437250-477 from 2778741 to 2944899 tid 3 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3147: read oid 243 snap -1 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: write oid 178 current snap is 358 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: seq_num 1472 ranges {737280=1} 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: writing mira0437250-178 from 737280 to 737281 tid 1 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3149: rollback oid 1 current snap is 358 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 1 to 304 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3150: snap_create 2014-07-01 17:11:16,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 1 to mira0437250-178 2014-07-01 17:11:16,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 3 to mira0437250-178 2014-07-01 17:11:16,891.891 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: finishing write tid 4 to mira0437250-178 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 178 v 1530 (ObjNum 1472 snap 358 seq_num 1472) dirty exists 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 1 v 792 (ObjNum 0 snap 0 seq_num 0) dirty exists 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: finishing write tid 1 to mira0437250-477 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: finishing write tid 2 to mira0437250-477 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: finishing write tid 3 to mira0437250-477 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: finishing write tid 5 to mira0437250-477 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: finishing write tid 6 to mira0437250-477 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 477 v 1963 (ObjNum 1471 snap 358 seq_num 1471) dirty exists 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: expect (ObjNum 137 snap 0 seq_num 137) 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: finishing write tid 6 to mira0435601-215 2014-07-01 17:11:16,892.892 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 349 (ObjNum 2238 snap 0 seq_num 2238) dirty exists 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: done (4 left) 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: done (3 left) 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: done (2 left) 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3659: done (1 left) 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3660: read oid 460 snap -1 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3661: read oid 42 snap -1 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3662: delete oid 109 current snap is 0 2014-07-01 17:11:16,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: expect (ObjNum 2024 snap 0 seq_num 2024) 2014-07-01 17:11:17,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: finishing write tid 6 to mira0437250-58 2014-07-01 17:11:17,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 1550 (ObjNum 1469 snap 358 seq_num 1469) dirty exists 2014-07-01 17:11:17,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3137: done (6 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3142: done (5 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3146: done (4 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3147: done (3 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3148: done (2 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3149: done (1 left) 2014-07-01 17:11:17,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3150: done (0 left) 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: read oid 67 snap -1 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3152: delete oid 432 current snap is 359 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3152: done (1 left) 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: read oid 354 snap -1 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: read oid 285 snap -1 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3155: snap_create 2014-07-01 17:11:17,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: expect (ObjNum 1399 snap 319 seq_num 1399) 2014-07-01 17:11:17,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: expect (ObjNum 1181 snap 250 seq_num 1181) 2014-07-01 17:11:18,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:3151: done (3 left) 2014-07-01 17:11:18,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:3153: done (2 left) 2014-07-01 17:11:18,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3154: done (1 left) 2014-07-01 17:11:18,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3155: done (0 left) 2014-07-01 17:11:18,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3156: snap_remove snap 340 2014-07-01 17:11:18,294.294 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:18,295.295 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:18,295.295 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_2 2014-07-01 17:11:18,296.296 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:11:18,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3660: expect (ObjNum 1713 snap 0 seq_num 1713) 2014-07-01 17:11:19,061.061 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:11:19,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:3156: done (0 left) 2014-07-01 17:11:19,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: write oid 366 current snap is 360 2014-07-01 17:11:19,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: seq_num 1474 ranges {129787=1} 2014-07-01 17:11:19,193.193 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: writing mira0437250-366 from 129787 to 129788 tid 1 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: read oid 162 snap -1 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3159: read oid 321 snap -1 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: read oid 278 snap 315 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: read oid 442 snap -1 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: read oid 426 snap -1 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: write oid 293 current snap is 360 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: seq_num 1475 ranges {438522=456109,1657542=698779,2438521=1} 2014-07-01 17:11:19,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: writing mira0437250-293 from 438522 to 894631 tid 1 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: writing mira0437250-293 from 1657542 to 2356321 tid 2 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: writing mira0437250-293 from 2438521 to 2438522 tid 3 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: read oid 400 snap -1 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: expect (ObjNum 1423 snap 331 seq_num 1423) 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: expect (ObjNum 1007 snap 196 seq_num 1007) 2014-07-01 17:11:19,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: expect (ObjNum 1271 snap 281 seq_num 1271) 2014-07-01 17:11:19,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: expect (ObjNum 991 snap 188 seq_num 991) 2014-07-01 17:11:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: finishing write tid 1 to mira0437250-366 2014-07-01 17:11:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: finishing write tid 3 to mira0437250-366 2014-07-01 17:11:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: finishing write tid 1 to mira0437250-293 2014-07-01 17:11:19,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: finishing write tid 2 to mira0437250-293 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: finishing write tid 3 to mira0437250-293 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: finishing write tid 5 to mira0437250-293 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: finishing write tid 4 to mira0437250-366 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 1097 (ObjNum 1474 snap 360 seq_num 1474) dirty exists 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: finishing write tid 6 to mira0437250-293 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 293 v 1590 (ObjNum 1475 snap 360 seq_num 1475) dirty exists 2014-07-01 17:11:19,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: expect (ObjNum 277 snap 0 seq_num 277) 2014-07-01 17:11:21,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:3661: expect (ObjNum 1969 snap 0 seq_num 1969) 2014-07-01 17:11:21,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3157: done (7 left) 2014-07-01 17:11:21,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3158: done (6 left) 2014-07-01 17:11:21,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3159: done (5 left) 2014-07-01 17:11:21,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3160: done (4 left) 2014-07-01 17:11:21,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3161: done (3 left) 2014-07-01 17:11:21,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3162: done (2 left) 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3163: done (1 left) 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3164: done (0 left) 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3165: delete oid 86 current snap is 360 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3165: done (0 left) 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: read oid 314 snap -1 2014-07-01 17:11:21,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: snap_create 2014-07-01 17:11:22,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3166: done (1 left) 2014-07-01 17:11:22,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3167: done (0 left) 2014-07-01 17:11:22,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3168: snap_remove snap 285 2014-07-01 17:11:22,646.646 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_2 pgp_num 66' 2014-07-01 17:11:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3168: done (0 left) 2014-07-01 17:11:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: rollback oid 401 current snap is 361 2014-07-01 17:11:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 401 to 348 2014-07-01 17:11:23,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: read oid 257 snap 353 2014-07-01 17:11:23,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: write oid 322 current snap is 361 2014-07-01 17:11:23,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: seq_num 1477 ranges {628981=552166,1601134=227847} 2014-07-01 17:11:23,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: writing mira0437250-322 from 628981 to 1181147 tid 1 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: writing mira0437250-322 from 1601134 to 1828981 tid 2 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3172: rollback oid 339 current snap is 361 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 339 to 325 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: read oid 150 snap -1 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3174: read oid 379 snap -1 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: write oid 434 current snap is 361 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: seq_num 1478 ranges {573811=559043,1598915=544436,2573810=1} 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: writing mira0437250-434 from 573811 to 1132854 tid 1 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: writing mira0437250-434 from 1598915 to 2143351 tid 2 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: writing mira0437250-434 from 2573810 to 2573811 tid 3 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: snap_create 2014-07-01 17:11:23,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: expect (ObjNum 256 snap 0 seq_num 256) 2014-07-01 17:11:23,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 1 to mira0437250-434 2014-07-01 17:11:23,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: expect (ObjNum 1183 snap 250 seq_num 1183) 2014-07-01 17:11:23,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: done (3 left) 2014-07-01 17:11:23,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3660: done (2 left) 2014-07-01 17:11:23,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3661: done (1 left) 2014-07-01 17:11:23,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3662: done (0 left) 2014-07-01 17:11:23,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: read oid 14 snap -1 2014-07-01 17:11:23,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 120 current snap is 0 2014-07-01 17:11:23,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: seq_num 2242 ranges {7561216=610304,8171520=802816,8974336=401408} 2014-07-01 17:11:23,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: writing mira0435601-120 from 7561216 to 8171520 tid 1 2014-07-01 17:11:23,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: writing mira0435601-120 from 8171520 to 8974336 tid 2 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: writing mira0435601-120 from 8974336 to 9375744 tid 3 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 305 current snap is 0 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: seq_num 2243 ranges {933888=634880} 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: writing mira0435601-305 from 933888 to 1568768 tid 1 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3666: read oid 399 snap -1 2014-07-01 17:11:23,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: expect (ObjNum 1569 snap 0 seq_num 1569) 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 2 to mira0437250-434 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 3 to mira0437250-434 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 5 to mira0437250-434 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: finishing write tid 6 to mira0437250-434 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 969 (ObjNum 1478 snap 361 seq_num 1478) dirty exists 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: finishing write tid 1 to mira0437250-322 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: finishing write tid 2 to mira0437250-322 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: finishing write tid 4 to mira0437250-322 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: finishing write tid 5 to mira0437250-322 2014-07-01 17:11:24,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 322 v 1053 (ObjNum 1477 snap 361 seq_num 1477) dirty exists 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 1221 (ObjNum 338 snap 0 seq_num 338) dirty exists 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 401 v 1267 (ObjNum 400 snap 0 seq_num 400) dirty exists 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3169: done (7 left) 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3170: done (6 left) 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3171: done (5 left) 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3172: done (4 left) 2014-07-01 17:11:24,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3173: done (3 left) 2014-07-01 17:11:24,274.274 INFO:teuthology.orchestra.run.mira079.stderr:set pool 16 pgp_num to 66 2014-07-01 17:11:24,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3174: done (2 left) 2014-07-01 17:11:24,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3175: done (1 left) 2014-07-01 17:11:24,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3176: done (0 left) 2014-07-01 17:11:24,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3177: snap_remove snap 336 2014-07-01 17:11:25,262.262 INFO:teuthology.task.rados.rados.0.mira043.stdout:3177: done (0 left) 2014-07-01 17:11:25,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: read oid 175 snap 356 2014-07-01 17:11:25,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: snap_create 2014-07-01 17:11:25,321.321 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 17:11:26,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3178: done (1 left) 2014-07-01 17:11:26,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3179: done (0 left) 2014-07-01 17:11:26,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3180: snap_remove snap 356 2014-07-01 17:11:27,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:3666: expect (ObjNum 2045 snap 0 seq_num 2045) 2014-07-01 17:11:27,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:3180: done (0 left) 2014-07-01 17:11:27,268.268 INFO:teuthology.task.rados.rados.0.mira043.stdout:3181: delete oid 210 current snap is 363 2014-07-01 17:11:27,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3181: done (0 left) 2014-07-01 17:11:27,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: snap_remove snap 353 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: finishing write tid 1 to mira0435601-120 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: finishing write tid 2 to mira0435601-120 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: finishing write tid 3 to mira0435601-120 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: finishing write tid 5 to mira0435601-120 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: finishing write tid 1 to mira0435601-305 2014-07-01 17:11:27,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: finishing write tid 3 to mira0435601-305 2014-07-01 17:11:27,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: finishing write tid 4 to mira0435601-305 2014-07-01 17:11:27,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 400 (ObjNum 2243 snap 0 seq_num 2243) dirty exists 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: finishing write tid 6 to mira0435601-120 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 168 (ObjNum 2242 snap 0 seq_num 2242) dirty exists 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: done (3 left) 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: done (2 left) 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: done (1 left) 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3666: done (0 left) 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 443 current snap is 0 2014-07-01 17:11:27,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: seq_num 2244 ranges {4018176=663552,4681728=409600,5091328=794624} 2014-07-01 17:11:27,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: writing mira0435601-443 from 4018176 to 4681728 tid 1 2014-07-01 17:11:27,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: writing mira0435601-443 from 4681728 to 5091328 tid 2 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: writing mira0435601-443 from 5091328 to 5885952 tid 3 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3668: read oid 476 snap -1 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: finishing write tid 1 to mira0435601-443 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: finishing write tid 2 to mira0435601-443 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3669: read oid 168 snap -1 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 498 current snap is 0 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: seq_num 2245 ranges {0=184320} 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: writing mira0435601-498 from 0 to 184320 tid 1 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 189 current snap is 0 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: seq_num 2246 ranges {5861376=53248} 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: writing mira0435601-189 from 5861376 to 5914624 tid 1 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 402 current snap is 0 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: seq_num 2247 ranges {2273280=483328,2756608=454656,3211264=348160} 2014-07-01 17:11:27,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: writing mira0435601-402 from 2273280 to 2756608 tid 1 2014-07-01 17:11:27,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: writing mira0435601-402 from 2756608 to 3211264 tid 2 2014-07-01 17:11:27,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: writing mira0435601-402 from 3211264 to 3559424 tid 3 2014-07-01 17:11:27,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 7 current snap is 0 2014-07-01 17:11:27,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: seq_num 2248 ranges {0=757760,757760=401408} 2014-07-01 17:11:27,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: writing mira0435601-7 from 0 to 757760 tid 1 2014-07-01 17:11:27,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: writing mira0435601-7 from 757760 to 1159168 tid 2 2014-07-01 17:11:27,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3674: delete oid 73 current snap is 0 2014-07-01 17:11:27,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3668: expect (ObjNum 2218 snap 0 seq_num 2218) 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: finishing write tid 1 to mira0435601-498 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: finishing write tid 3 to mira0435601-498 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: finishing write tid 3 to mira0435601-443 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: finishing write tid 5 to mira0435601-443 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: finishing write tid 4 to mira0435601-498 2014-07-01 17:11:27,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 498 v 347 (ObjNum 2245 snap 0 seq_num 2245) dirty exists 2014-07-01 17:11:27,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: finishing write tid 6 to mira0435601-443 2014-07-01 17:11:27,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 295 (ObjNum 2244 snap 0 seq_num 2244) dirty exists 2014-07-01 17:11:27,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 1 to mira0435601-189 2014-07-01 17:11:27,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: done (7 left) 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3668: done (6 left) 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3669: done (5 left) 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: done (4 left) 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3674: done (3 left) 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 137 current snap is 0 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: seq_num 2250 ranges {1597440=704512} 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: writing mira0435601-137 from 1597440 to 2301952 tid 1 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 182 current snap is 0 2014-07-01 17:11:27,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: seq_num 2251 ranges {0=573440} 2014-07-01 17:11:28,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: writing mira0435601-182 from 0 to 573440 tid 1 2014-07-01 17:11:28,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: read oid 496 snap -1 2014-07-01 17:11:28,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 3 to mira0435601-189 2014-07-01 17:11:28,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 4 to mira0435601-189 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 276 (ObjNum 2246 snap 0 seq_num 2246) dirty exists 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: finishing write tid 1 to mira0435601-402 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: finishing write tid 2 to mira0435601-402 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: finishing write tid 3 to mira0435601-402 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: finishing write tid 5 to mira0435601-402 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: done (5 left) 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 157 current snap is 0 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: seq_num 2252 ranges {5595136=446464} 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: writing mira0435601-157 from 5595136 to 6041600 tid 1 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3679: read oid 245 snap -1 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: finishing write tid 6 to mira0435601-402 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 349 (ObjNum 2247 snap 0 seq_num 2247) dirty exists 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: finishing write tid 1 to mira0435601-7 2014-07-01 17:11:28,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: finishing write tid 2 to mira0435601-7 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: finishing write tid 4 to mira0435601-7 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: finishing write tid 5 to mira0435601-7 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 7 v 234 (ObjNum 2248 snap 0 seq_num 2248) dirty exists 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: finishing write tid 1 to mira0435601-137 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: finishing write tid 3 to mira0435601-137 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: finishing write tid 4 to mira0435601-137 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 137 v 159 (ObjNum 2250 snap 0 seq_num 2250) dirty exists 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: finishing write tid 1 to mira0435601-182 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: finishing write tid 3 to mira0435601-182 2014-07-01 17:11:28,071.071 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: finishing write tid 4 to mira0435601-182 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 293 (ObjNum 2251 snap 0 seq_num 2251) dirty exists 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: done (6 left) 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: done (5 left) 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: done (4 left) 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: done (3 left) 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 120 current snap is 0 2014-07-01 17:11:28,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: seq_num 2253 ranges {9375744=475136,9850880=602112,10452992=569344,11022336=32768} 2014-07-01 17:11:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: writing mira0435601-120 from 9375744 to 9850880 tid 1 2014-07-01 17:11:28,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: writing mira0435601-120 from 9850880 to 10452992 tid 2 2014-07-01 17:11:28,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: writing mira0435601-120 from 10452992 to 11022336 tid 3 2014-07-01 17:11:28,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: writing mira0435601-120 from 11022336 to 11055104 tid 4 2014-07-01 17:11:28,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: delete oid 444 current snap is 0 2014-07-01 17:11:28,234.234 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: expect (ObjNum 1230 snap 0 seq_num 1230) 2014-07-01 17:11:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:3182: done (0 left) 2014-07-01 17:11:28,287.287 INFO:teuthology.task.rados.rados.0.mira043.stdout:3183: rollback oid 166 current snap is 363 2014-07-01 17:11:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 166 to 349 2014-07-01 17:11:28,288.288 INFO:teuthology.task.rados.rados.0.mira043.stdout:3184: delete oid 237 current snap is 363 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3184: done (1 left) 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: rollback oid 138 current snap is 363 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 138 to 330 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: write oid 423 current snap is 363 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: seq_num 1481 ranges {275051=1} 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: writing mira0437250-423 from 275051 to 275052 tid 1 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: snap_remove snap 338 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 648 (ObjNum 1169 snap 246 seq_num 1169) dirty exists 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 138 v 841 (ObjNum 137 snap 0 seq_num 137) dirty exists 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 1 to mira0437250-423 2014-07-01 17:11:28,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 3 to mira0437250-423 2014-07-01 17:11:28,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: finishing write tid 4 to mira0437250-423 2014-07-01 17:11:28,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 423 v 1054 (ObjNum 1481 snap 363 seq_num 1481) dirty exists 2014-07-01 17:11:29,290.290 INFO:teuthology.task.thrashosds.thrasher:in_osds: [5, 4, 0, 2, 1, 3] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:29,290.290 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:29,291.291 INFO:teuthology.task.thrashosds.thrasher:Removing osd 5, in_osds are: [5, 4, 0, 2, 1, 3] 2014-07-01 17:11:29,291.291 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 5' 2014-07-01 17:11:29,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3183: done (3 left) 2014-07-01 17:11:29,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3185: done (2 left) 2014-07-01 17:11:29,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3186: done (1 left) 2014-07-01 17:11:29,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3187: done (0 left) 2014-07-01 17:11:29,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3188: delete oid 259 current snap is 363 2014-07-01 17:11:29,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3188: done (0 left) 2014-07-01 17:11:29,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3189: rollback oid 243 current snap is 363 2014-07-01 17:11:29,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 243 to 344 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: write oid 344 current snap is 363 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: seq_num 1483 ranges {563688=607632,1645159=118529} 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0437250-344 from 563688 to 1171320 tid 1 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: writing mira0437250-344 from 1645159 to 1763688 tid 2 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: write oid 312 current snap is 363 2014-07-01 17:11:29,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: seq_num 1484 ranges {776294=730861,1976293=1} 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: writing mira0437250-312 from 776294 to 1507155 tid 1 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: writing mira0437250-312 from 1976293 to 1976294 tid 2 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: delete oid 339 current snap is 363 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 0 (ObjNum 30 snap 0 seq_num 1) dirty dne 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 1 to mira0437250-344 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 2 to mira0437250-344 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 4 to mira0437250-344 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: finishing write tid 5 to mira0437250-344 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 344 v 1176 (ObjNum 1483 snap 363 seq_num 1483) dirty exists 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3189: done (3 left) 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3190: done (2 left) 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3192: done (1 left) 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3193: snap_create 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 1 to mira0437250-312 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 2 to mira0437250-312 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 4 to mira0437250-312 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: finishing write tid 5 to mira0437250-312 2014-07-01 17:11:29,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 312 v 1420 (ObjNum 1484 snap 363 seq_num 1484) dirty exists 2014-07-01 17:11:30,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3191: done (1 left) 2014-07-01 17:11:30,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3193: done (0 left) 2014-07-01 17:11:30,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: read oid 146 snap -1 2014-07-01 17:11:30,340.340 INFO:teuthology.task.rados.rados.0.mira043.stdout:3195: snap_create 2014-07-01 17:11:30,365.365 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.5. 2014-07-01 17:11:31,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: finishing write tid 1 to mira0435601-157 2014-07-01 17:11:31,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: finishing write tid 3 to mira0435601-157 2014-07-01 17:11:31,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: finishing write tid 4 to mira0435601-157 2014-07-01 17:11:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 310 (ObjNum 2252 snap 0 seq_num 2252) dirty exists 2014-07-01 17:11:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 1 to mira0435601-120 2014-07-01 17:11:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 2 to mira0435601-120 2014-07-01 17:11:31,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 3 to mira0435601-120 2014-07-01 17:11:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 4 to mira0435601-120 2014-07-01 17:11:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 6 to mira0435601-120 2014-07-01 17:11:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: finishing write tid 7 to mira0435601-120 2014-07-01 17:11:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 120 v 173 (ObjNum 2253 snap 0 seq_num 2253) dirty exists 2014-07-01 17:11:31,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: done (4 left) 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: done (3 left) 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3679: done (2 left) 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: done (1 left) 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: done (0 left) 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 264 current snap is 0 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: seq_num 2255 ranges {3194880=667648,3862528=663552,4526080=671744,5197824=270336} 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0435601-264 from 3194880 to 3862528 tid 1 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0435601-264 from 3862528 to 4526080 tid 2 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0435601-264 from 4526080 to 5197824 tid 3 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0435601-264 from 5197824 to 5468160 tid 4 2014-07-01 17:11:31,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3195: done (1 left) 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: read oid 481 snap -1 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: rollback oid 223 current snap is 365 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 223 to 304 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: write oid 357 current snap is 365 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: seq_num 1486 ranges {663905=674469,1915364=643918,3118978=344927} 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0437250-357 from 663905 to 1338374 tid 1 2014-07-01 17:11:31,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0437250-357 from 1915364 to 2559282 tid 2 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 336 current snap is 0 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: seq_num 2256 ranges {2969600=696320,3665920=401408} 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: writing mira0435601-336 from 2969600 to 3665920 tid 1 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: writing mira0435601-336 from 3665920 to 4067328 tid 2 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 108 current snap is 0 2014-07-01 17:11:31,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: seq_num 2257 ranges {421888=565248,987136=401408} 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: writing mira0435601-108 from 421888 to 987136 tid 1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: writing mira0437250-357 from 3118978 to 3463905 tid 3 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: read oid 72 snap -1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: write oid 60 current snap is 365 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: seq_num 1487 ranges {759135=530946,2028473=465537,3070604=88531} 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: writing mira0437250-60 from 759135 to 1290081 tid 1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: writing mira0437250-60 from 2028473 to 2494010 tid 2 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: writing mira0437250-60 from 3070604 to 3159135 tid 3 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: read oid 292 snap -1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3202: read oid 365 snap -1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: read oid 435 snap -1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: write oid 167 current snap is 365 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: seq_num 1488 ranges {420316=1} 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: writing mira0437250-167 from 420316 to 420317 tid 1 2014-07-01 17:11:31,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3205: snap_remove snap 333 2014-07-01 17:11:31,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: writing mira0435601-108 from 987136 to 1388544 tid 2 2014-07-01 17:11:31,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3685: read oid 76 snap -1 2014-07-01 17:11:31,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3686: delete oid 411 current snap is 0 2014-07-01 17:11:31,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: expect (ObjNum 700 snap 92 seq_num 700) 2014-07-01 17:11:31,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3685: expect (ObjNum 2174 snap 0 seq_num 2174) 2014-07-01 17:11:32,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: finishing write tid 1 to mira0437250-60 2014-07-01 17:11:32,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: finishing write tid 2 to mira0437250-60 2014-07-01 17:11:32,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3202: expect (ObjNum 850 snap 141 seq_num 850) 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 223 v 928 (ObjNum 796 snap 127 seq_num 796) dirty exists 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: finishing write tid 3 to mira0437250-60 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 1 to mira0437250-357 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 2 to mira0437250-357 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 3 to mira0437250-357 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 5 to mira0437250-357 2014-07-01 17:11:32,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: finishing write tid 6 to mira0437250-357 2014-07-01 17:11:32,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 1670 (ObjNum 1486 snap 365 seq_num 1486) dirty exists 2014-07-01 17:11:32,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: finishing write tid 5 to mira0437250-60 2014-07-01 17:11:32,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: finishing write tid 6 to mira0437250-60 2014-07-01 17:11:32,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 60 v 875 (ObjNum 1487 snap 365 seq_num 1487) dirty exists 2014-07-01 17:11:32,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: expect (ObjNum 693 snap 88 seq_num 693) 2014-07-01 17:11:33,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: expect (ObjNum 1435 snap 342 seq_num 1435) 2014-07-01 17:11:33,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: finishing write tid 1 to mira0437250-167 2014-07-01 17:11:33,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: expect (ObjNum 1125 snap 228 seq_num 1125) 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: finishing write tid 3 to mira0437250-167 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: finishing write tid 4 to mira0437250-167 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 167 v 878 (ObjNum 1488 snap 365 seq_num 1488) dirty exists 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3194: done (10 left) 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3196: done (9 left) 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3197: done (8 left) 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3198: done (7 left) 2014-07-01 17:11:34,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3199: done (6 left) 2014-07-01 17:11:34,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:3200: done (5 left) 2014-07-01 17:11:34,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:3201: done (4 left) 2014-07-01 17:11:34,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:3202: done (3 left) 2014-07-01 17:11:34,286.286 INFO:teuthology.task.rados.rados.0.mira043.stdout:3203: done (2 left) 2014-07-01 17:11:34,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3204: done (1 left) 2014-07-01 17:11:34,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3205: done (0 left) 2014-07-01 17:11:34,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3206: read oid 53 snap -1 2014-07-01 17:11:34,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: snap_create 2014-07-01 17:11:34,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3206: expect (ObjNum 1388 snap 315 seq_num 1388) 2014-07-01 17:11:34,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 1 to mira0435601-264 2014-07-01 17:11:34,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 2 to mira0435601-264 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 3 to mira0435601-264 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 4 to mira0435601-264 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 6 to mira0435601-264 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 7 to mira0435601-264 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 383 (ObjNum 2255 snap 0 seq_num 2255) dirty exists 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: finishing write tid 1 to mira0435601-108 2014-07-01 17:11:34,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: finishing write tid 2 to mira0435601-108 2014-07-01 17:11:34,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: finishing write tid 4 to mira0435601-108 2014-07-01 17:11:34,557.557 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: finishing write tid 5 to mira0435601-108 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 108 v 272 (ObjNum 2257 snap 0 seq_num 2257) dirty exists 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: done (4 left) 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: done (3 left) 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3685: done (2 left) 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3686: done (1 left) 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 411 current snap is 0 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: seq_num 2259 ranges {0=516096} 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: writing mira0435601-411 from 0 to 516096 tid 1 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: read oid 95 snap -1 2014-07-01 17:11:34,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 17:11:34,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: seq_num 2260 ranges {7118848=385024} 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: writing mira0435601-226 from 7118848 to 7503872 tid 1 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: seq_num 2261 ranges {4964352=413696,5378048=409600,5787648=462848,6250496=331776} 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: writing mira0435601-14 from 4964352 to 5378048 tid 1 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: writing mira0435601-14 from 5378048 to 5787648 tid 2 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: writing mira0435601-14 from 5787648 to 6250496 tid 3 2014-07-01 17:11:34,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: writing mira0435601-14 from 6250496 to 6582272 tid 4 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: read oid 327 snap -1 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 1 to mira0435601-14 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: finishing write tid 1 to mira0435601-411 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: finishing write tid 3 to mira0435601-411 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: done (5 left) 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3692: read oid 437 snap -1 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 17:11:34,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: seq_num 2262 ranges {2420736=688128,3108864=794624,3903488=8192} 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: writing mira0435601-124 from 2420736 to 3108864 tid 1 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: writing mira0435601-124 from 3108864 to 3903488 tid 2 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: writing mira0435601-124 from 3903488 to 3911680 tid 3 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 494 current snap is 0 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: seq_num 2263 ranges {4796416=557056,5353472=626688,5980160=176128} 2014-07-01 17:11:34,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: writing mira0435601-494 from 4796416 to 5353472 tid 1 2014-07-01 17:11:34,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: writing mira0435601-494 from 5353472 to 5980160 tid 2 2014-07-01 17:11:34,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: writing mira0435601-494 from 5980160 to 6156288 tid 3 2014-07-01 17:11:34,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: read oid 282 snap -1 2014-07-01 17:11:34,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 474 current snap is 0 2014-07-01 17:11:34,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: seq_num 2264 ranges {3600384=634880,4235264=401408} 2014-07-01 17:11:34,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: writing mira0435601-474 from 3600384 to 4235264 tid 1 2014-07-01 17:11:34,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: writing mira0435601-474 from 4235264 to 4636672 tid 2 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 297 current snap is 0 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: seq_num 2265 ranges {282624=507904,790528=401408} 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: writing mira0435601-297 from 282624 to 790528 tid 1 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: writing mira0435601-297 from 790528 to 1191936 tid 2 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 94 current snap is 0 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: seq_num 2266 ranges {0=778240} 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: writing mira0435601-94 from 0 to 778240 tid 1 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3699: delete oid 338 current snap is 0 2014-07-01 17:11:34,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: expect (ObjNum 2057 snap 0 seq_num 2057) 2014-07-01 17:11:35,384.384 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:35,384.384 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:35,384.384 INFO:teuthology.task.thrashosds.thrasher:Killing osd 5, live_osds are [1, 0, 2, 4, 5, 3] 2014-07-01 17:11:35,386.386 DEBUG:teuthology.task.ceph.osd.5:waiting for process to exit 2014-07-01 17:11:35,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3206: done (1 left) 2014-07-01 17:11:35,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3207: done (0 left) 2014-07-01 17:11:35,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3208: delete oid 451 current snap is 366 2014-07-01 17:11:35,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3208: done (0 left) 2014-07-01 17:11:35,538.538 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: snap_remove snap 360 2014-07-01 17:11:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3209: done (0 left) 2014-07-01 17:11:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: read oid 53 snap -1 2014-07-01 17:11:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: write oid 289 current snap is 366 2014-07-01 17:11:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: seq_num 1490 ranges {460376=400000} 2014-07-01 17:11:35,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: writing mira0437250-289 from 460376 to 860376 tid 1 2014-07-01 17:11:35,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: snap_remove snap 358 2014-07-01 17:11:35,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: expect (ObjNum 1388 snap 315 seq_num 1388) 2014-07-01 17:11:37,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 1 to mira0437250-289 2014-07-01 17:11:37,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 3 to mira0437250-289 2014-07-01 17:11:37,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: finishing write tid 4 to mira0437250-289 2014-07-01 17:11:37,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 289 v 962 (ObjNum 1490 snap 366 seq_num 1490) dirty exists 2014-07-01 17:11:39,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 1 to mira0435601-226 2014-07-01 17:11:39,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 3 to mira0435601-226 2014-07-01 17:11:39,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:3692: expect (ObjNum 2033 snap 0 seq_num 2033) 2014-07-01 17:11:40,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 2 to mira0435601-14 2014-07-01 17:11:40,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: expect (ObjNum 2081 snap 0 seq_num 2081) 2014-07-01 17:11:40,548.548 INFO:teuthology.task.rados.rados.0.mira043.stdout:3210: done (2 left) 2014-07-01 17:11:40,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3211: done (1 left) 2014-07-01 17:11:40,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3212: done (0 left) 2014-07-01 17:11:40,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3213: delete oid 220 current snap is 366 2014-07-01 17:11:40,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3213: done (0 left) 2014-07-01 17:11:40,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3214: delete oid 189 current snap is 366 2014-07-01 17:11:40,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3214: done (0 left) 2014-07-01 17:11:40,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3215: snap_create 2014-07-01 17:11:41,384.384 INFO:teuthology.task.ceph.osd.5:Stopped 2014-07-01 17:11:41,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3215: done (0 left) 2014-07-01 17:11:41,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: write oid 204 current snap is 367 2014-07-01 17:11:41,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: seq_num 1493 ranges {560593=700716,1806001=754592} 2014-07-01 17:11:41,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: writing mira0437250-204 from 560593 to 1261309 tid 1 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: writing mira0437250-204 from 1806001 to 2560593 tid 2 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: read oid 131 snap -1 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: read oid 252 snap 366 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: rollback oid 79 current snap is 367 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 79 to 352 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: snap_create 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 0 (ObjNum 3 snap 0 seq_num 1) dirty dne 2014-07-01 17:11:41,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: expect (ObjNum 1453 snap 350 seq_num 1453) 2014-07-01 17:11:42,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 1 to mira0437250-204 2014-07-01 17:11:42,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 2 to mira0437250-204 2014-07-01 17:11:42,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 4 to mira0437250-204 2014-07-01 17:11:42,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: finishing write tid 5 to mira0437250-204 2014-07-01 17:11:42,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 1414 (ObjNum 1493 snap 367 seq_num 1493) dirty exists 2014-07-01 17:11:42,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3216: done (4 left) 2014-07-01 17:11:42,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3217: done (3 left) 2014-07-01 17:11:42,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3218: done (2 left) 2014-07-01 17:11:42,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3219: done (1 left) 2014-07-01 17:11:42,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3220: done (0 left) 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: read oid 402 snap -1 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: write oid 199 current snap is 368 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: seq_num 1494 ranges {656847=424968,1671557=458728,2256846=1} 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: writing mira0437250-199 from 656847 to 1081815 tid 1 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: writing mira0437250-199 from 1671557 to 2130285 tid 2 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: writing mira0437250-199 from 2256846 to 2256847 tid 3 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: rollback oid 235 current snap is 368 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 235 to 325 2014-07-01 17:11:43,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: snap_remove snap 367 2014-07-01 17:11:43,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3224: done (3 left) 2014-07-01 17:11:43,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: write oid 303 current snap is 368 2014-07-01 17:11:43,646.646 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: seq_num 1495 ranges {49228=1} 2014-07-01 17:11:43,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: writing mira0437250-303 from 49228 to 49229 tid 1 2014-07-01 17:11:43,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: rollback oid 63 current snap is 368 2014-07-01 17:11:43,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 63 to 326 2014-07-01 17:11:43,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3227: delete oid 291 current snap is 368 2014-07-01 17:11:43,848.848 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: expect (ObjNum 1451 snap 349 seq_num 1451) 2014-07-01 17:11:44,417.417 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 235 v 0 (ObjNum 0 snap 0 seq_num 32) dirty dne 2014-07-01 17:11:44,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 1 to mira0437250-199 2014-07-01 17:11:44,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 2 to mira0437250-199 2014-07-01 17:11:44,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 3 to mira0437250-199 2014-07-01 17:11:44,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 5 to mira0437250-199 2014-07-01 17:11:44,418.418 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: finishing write tid 6 to mira0437250-199 2014-07-01 17:11:44,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 854 (ObjNum 1494 snap 368 seq_num 1494) dirty exists 2014-07-01 17:11:44,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 63 v 1114 (ObjNum 62 snap 0 seq_num 62) dirty exists 2014-07-01 17:11:44,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3221: done (5 left) 2014-07-01 17:11:44,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3222: done (4 left) 2014-07-01 17:11:44,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3223: done (3 left) 2014-07-01 17:11:44,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3226: done (2 left) 2014-07-01 17:11:44,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3227: done (1 left) 2014-07-01 17:11:44,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: read oid 324 snap 314 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3229: snap_create 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: finishing write tid 1 to mira0437250-303 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: finishing write tid 3 to mira0437250-303 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: finishing write tid 4 to mira0437250-303 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 1715 (ObjNum 1495 snap 368 seq_num 1495) dirty exists 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3225: done (2 left) 2014-07-01 17:11:44,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3228: done (1 left) 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3229: done (0 left) 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: rollback oid 27 current snap is 369 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 272 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: read oid 305 snap -1 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3232: read oid 237 snap -1 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: read oid 268 snap -1 2014-07-01 17:11:44,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: delete oid 488 current snap is 369 2014-07-01 17:11:44,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: expect (ObjNum 1234 snap 269 seq_num 1234) 2014-07-01 17:11:45,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3231: done (4 left) 2014-07-01 17:11:45,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3232: done (3 left) 2014-07-01 17:11:45,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3233: done (2 left) 2014-07-01 17:11:45,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:3234: done (1 left) 2014-07-01 17:11:45,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: read oid 394 snap 330 2014-07-01 17:11:45,767.767 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: write oid 487 current snap is 369 2014-07-01 17:11:45,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: seq_num 1498 ranges {550470=748281,1750469=1} 2014-07-01 17:11:45,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: writing mira0437250-487 from 550470 to 1298751 tid 1 2014-07-01 17:11:45,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: writing mira0437250-487 from 1750469 to 1750470 tid 2 2014-07-01 17:11:45,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: delete oid 57 current snap is 369 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 1602 (ObjNum 647 snap 67 seq_num 647) dirty exists 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 1 to mira0437250-487 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 2 to mira0437250-487 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 4 to mira0437250-487 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: finishing write tid 5 to mira0437250-487 2014-07-01 17:11:45,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 487 v 1123 (ObjNum 1498 snap 369 seq_num 1498) dirty exists 2014-07-01 17:11:46,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3230: done (3 left) 2014-07-01 17:11:46,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3236: done (2 left) 2014-07-01 17:11:46,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3237: done (1 left) 2014-07-01 17:11:46,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: write oid 296 current snap is 369 2014-07-01 17:11:46,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: seq_num 1500 ranges {741953=400000} 2014-07-01 17:11:46,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: writing mira0437250-296 from 741953 to 1141953 tid 1 2014-07-01 17:11:46,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: snap_remove snap 314 2014-07-01 17:11:46,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: expect (ObjNum 990 snap 188 seq_num 990) 2014-07-01 17:11:46,385.385 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3] out_osds: [5] dead_osds: [5] live_osds: [1, 0, 2, 4, 3] 2014-07-01 17:11:46,385.385 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:46,385.385 INFO:teuthology.task.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2014-07-01 17:11:46,385.385 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 4 0' 2014-07-01 17:11:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: finishing write tid 4 to mira0435601-411 2014-07-01 17:11:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 223 (ObjNum 2259 snap 0 seq_num 2259) dirty exists 2014-07-01 17:11:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 3 to mira0435601-14 2014-07-01 17:11:46,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 4 to mira0435601-14 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 6 to mira0435601-14 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 1 to mira0435601-494 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 4 to mira0435601-226 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 143 (ObjNum 2260 snap 0 seq_num 2260) dirty exists 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: finishing write tid 1 to mira0435601-474 2014-07-01 17:11:46,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: done (12 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: done (11 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: done (10 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3692: done (9 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: done (8 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3699: done (7 left) 2014-07-01 17:11:46,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 395 current snap is 0 2014-07-01 17:11:47,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: seq_num 2268 ranges {3715072=483328,4198400=786432,4984832=417792} 2014-07-01 17:11:47,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: writing mira0435601-395 from 3715072 to 4198400 tid 1 2014-07-01 17:11:47,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: writing mira0435601-395 from 4198400 to 4984832 tid 2 2014-07-01 17:11:47,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: writing mira0435601-395 from 4984832 to 5402624 tid 3 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 168 current snap is 0 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: seq_num 2269 ranges {0=196608} 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: writing mira0435601-168 from 0 to 196608 tid 1 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 306 current snap is 0 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: seq_num 2270 ranges {0=626688,626688=602112,1228800=200704} 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: writing mira0435601-306 from 0 to 626688 tid 1 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: writing mira0435601-306 from 626688 to 1228800 tid 2 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: writing mira0435601-306 from 1228800 to 1429504 tid 3 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 204 current snap is 0 2014-07-01 17:11:47,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: seq_num 2271 ranges {0=249856} 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: writing mira0435601-204 from 0 to 249856 tid 1 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: delete oid 302 current snap is 0 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: finishing write tid 2 to mira0435601-474 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: finishing write tid 4 to mira0435601-474 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 7 to mira0435601-14 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 169 (ObjNum 2261 snap 0 seq_num 2261) dirty exists 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 2 to mira0435601-494 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 3 to mira0435601-494 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: finishing write tid 5 to mira0435601-474 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 226 (ObjNum 2264 snap 0 seq_num 2264) dirty exists 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 5 to mira0435601-494 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 1 to mira0435601-94 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 3 to mira0435601-94 2014-07-01 17:11:47,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 6 to mira0435601-494 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 314 (ObjNum 2263 snap 0 seq_num 2263) dirty exists 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 4 to mira0435601-94 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 264 (ObjNum 2266 snap 0 seq_num 2266) dirty exists 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 1 to mira0435601-336 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 2 to mira0435601-336 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 4 to mira0435601-336 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 1 to mira0435601-297 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 2 to mira0435601-297 2014-07-01 17:11:47,100.100 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 4 to mira0435601-297 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 5 to mira0435601-336 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 336 v 194 (ObjNum 2256 snap 0 seq_num 2256) dirty exists 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: finishing write tid 1 to mira0435601-124 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: finishing write tid 2 to mira0435601-124 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: finishing write tid 3 to mira0435601-124 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: finishing write tid 5 to mira0435601-124 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: finishing write tid 6 to mira0435601-124 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 353 (ObjNum 2262 snap 0 seq_num 2262) dirty exists 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 5 to mira0435601-297 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 297 v 225 (ObjNum 2265 snap 0 seq_num 2265) dirty exists 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: finishing write tid 1 to mira0435601-168 2014-07-01 17:11:47,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: finishing write tid 3 to mira0435601-168 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: finishing write tid 4 to mira0435601-168 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 262 (ObjNum 2269 snap 0 seq_num 2269) dirty exists 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: done (11 left) 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: done (10 left) 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: done (9 left) 2014-07-01 17:11:47,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: done (8 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: done (7 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: done (6 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: done (5 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: done (4 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: done (3 left) 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 310 current snap is 0 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: seq_num 2273 ranges {0=446464,446464=401408} 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: writing mira0435601-310 from 0 to 446464 tid 1 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: writing mira0435601-310 from 446464 to 847872 tid 2 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 389 current snap is 0 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: seq_num 2274 ranges {1228800=471040,1699840=581632,2281472=630784,2912256=393216} 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: writing mira0435601-389 from 1228800 to 1699840 tid 1 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: writing mira0435601-389 from 1699840 to 2281472 tid 2 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: writing mira0435601-389 from 2281472 to 2912256 tid 3 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: writing mira0435601-389 from 2912256 to 3305472 tid 4 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3707: read oid 428 snap -1 2014-07-01 17:11:47,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: finishing write tid 1 to mira0435601-306 2014-07-01 17:11:47,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: finishing write tid 2 to mira0435601-306 2014-07-01 17:11:47,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: finishing write tid 3 to mira0435601-306 2014-07-01 17:11:47,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: finishing write tid 5 to mira0435601-306 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: finishing write tid 6 to mira0435601-306 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 306 v 297 (ObjNum 2270 snap 0 seq_num 2270) dirty exists 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: done (5 left) 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3708: read oid 366 snap -1 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 14 current snap is 0 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: seq_num 2275 ranges {6582272=589824} 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: writing mira0435601-14 from 6582272 to 7172096 tid 1 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3710: read oid 456 snap -1 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: finishing write tid 1 to mira0435601-310 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: finishing write tid 2 to mira0435601-310 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: finishing write tid 4 to mira0435601-310 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 1 to mira0435601-204 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 3 to mira0435601-204 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: finishing write tid 1 to mira0435601-395 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: finishing write tid 5 to mira0435601-310 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 352 (ObjNum 2273 snap 0 seq_num 2273) dirty exists 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: done (7 left) 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: read oid 468 snap -1 2014-07-01 17:11:47,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: read oid 61 snap -1 2014-07-01 17:11:47,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 189 current snap is 0 2014-07-01 17:11:47,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: seq_num 2276 ranges {5914624=618496,6533120=643072,7176192=561152} 2014-07-01 17:11:47,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: writing mira0435601-189 from 5914624 to 6533120 tid 1 2014-07-01 17:11:47,323.323 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: writing mira0435601-189 from 6533120 to 7176192 tid 2 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: writing mira0435601-189 from 7176192 to 7737344 tid 3 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 444 current snap is 0 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: seq_num 2277 ranges {0=692224,692224=421888,1114112=380928} 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: writing mira0435601-444 from 0 to 692224 tid 1 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: writing mira0435601-444 from 692224 to 1114112 tid 2 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: writing mira0435601-444 from 1114112 to 1495040 tid 3 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3715: read oid 233 snap -1 2014-07-01 17:11:47,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3707: expect (ObjNum 1327 snap 0 seq_num 1327) 2014-07-01 17:11:47,792.792 INFO:teuthology.orchestra.run.mira079.stderr:set osd.4 primary-affinity to 0 (802) 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 1 to mira0437250-296 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 3 to mira0437250-296 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: finishing write tid 4 to mira0437250-296 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 1135 (ObjNum 1500 snap 369 seq_num 1500) dirty exists 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3235: done (2 left) 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3238: done (1 left) 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3239: done (0 left) 2014-07-01 17:11:47,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: read oid 293 snap -1 2014-07-01 17:11:47,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: snap_create 2014-07-01 17:11:47,856.856 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: expect (ObjNum 1475 snap 360 seq_num 1475) 2014-07-01 17:11:48,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: finishing write tid 2 to mira0435601-395 2014-07-01 17:11:48,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: finishing write tid 3 to mira0435601-395 2014-07-01 17:11:48,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: expect (ObjNum 689 snap 0 seq_num 689) 2014-07-01 17:11:49,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3240: done (1 left) 2014-07-01 17:11:49,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3241: done (0 left) 2014-07-01 17:11:49,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: read oid 157 snap -1 2014-07-01 17:11:49,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: rollback oid 187 current snap is 370 2014-07-01 17:11:49,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 187 to 355 2014-07-01 17:11:49,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: delete oid 183 current snap is 370 2014-07-01 17:11:49,474.474 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: finishing write tid 5 to mira0435601-395 2014-07-01 17:11:49,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: finishing write tid 6 to mira0435601-395 2014-07-01 17:11:49,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 395 v 268 (ObjNum 2268 snap 0 seq_num 2268) dirty exists 2014-07-01 17:11:49,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 4 to mira0435601-204 2014-07-01 17:11:49,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 243 (ObjNum 2271 snap 0 seq_num 2271) dirty exists 2014-07-01 17:11:49,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3708: expect (ObjNum 2236 snap 0 seq_num 2236) 2014-07-01 17:11:49,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3244: done (2 left) 2014-07-01 17:11:49,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3245: snap_remove snap 345 2014-07-01 17:11:49,578.578 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 1430 (ObjNum 1063 snap 211 seq_num 1063) dirty exists 2014-07-01 17:11:49,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: expect (ObjNum 1432 snap 341 seq_num 1432) 2014-07-01 17:11:51,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 1 to mira0435601-14 2014-07-01 17:11:51,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3715: expect (ObjNum 1811 snap 0 seq_num 1811) 2014-07-01 17:11:51,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3242: done (2 left) 2014-07-01 17:11:51,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3243: done (1 left) 2014-07-01 17:11:51,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3245: done (0 left) 2014-07-01 17:11:51,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3246: read oid 475 snap -1 2014-07-01 17:11:51,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3247: delete oid 400 current snap is 370 2014-07-01 17:11:51,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3246: expect (ObjNum 1318 snap 302 seq_num 1318) 2014-07-01 17:11:51,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3246: done (1 left) 2014-07-01 17:11:51,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3247: done (0 left) 2014-07-01 17:11:51,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: rollback oid 83 current snap is 370 2014-07-01 17:11:51,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 83 to 330 2014-07-01 17:11:51,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: snap_remove snap 365 2014-07-01 17:11:51,837.837 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 660 (ObjNum 1077 snap 216 seq_num 1077) dirty exists 2014-07-01 17:11:51,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3248: done (1 left) 2014-07-01 17:11:51,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3249: done (0 left) 2014-07-01 17:11:51,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3250: delete oid 69 current snap is 370 2014-07-01 17:11:51,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3250: done (0 left) 2014-07-01 17:11:51,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3251: snap_remove snap 362 2014-07-01 17:11:52,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3251: done (0 left) 2014-07-01 17:11:52,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:3252: read oid 140 snap -1 2014-07-01 17:11:52,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: write oid 282 current snap is 370 2014-07-01 17:11:52,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: seq_num 1504 ranges {787000=401699,1697870=733671,3197738=389262} 2014-07-01 17:11:52,801.801 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3] out_osds: [5] dead_osds: [5] live_osds: [1, 0, 2, 4, 3] 2014-07-01 17:11:52,801.801 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:52,801.801 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 5 2014-07-01 17:11:52,802.802 INFO:teuthology.task.ceph.osd.5:Restarting daemon 2014-07-01 17:11:52,802.802 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 5' 2014-07-01 17:11:52,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: writing mira0437250-282 from 787000 to 1188699 tid 1 2014-07-01 17:11:52,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: writing mira0437250-282 from 1697870 to 2431541 tid 2 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: writing mira0437250-282 from 3197738 to 3587000 tid 3 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3254: read oid 372 snap -1 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3252: done (2 left) 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: snap_create 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: finishing write tid 1 to mira0437250-282 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: finishing write tid 2 to mira0437250-282 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: finishing write tid 3 to mira0437250-282 2014-07-01 17:11:52,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: finishing write tid 5 to mira0437250-282 2014-07-01 17:11:52,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: finishing write tid 6 to mira0437250-282 2014-07-01 17:11:52,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 282 v 1138 (ObjNum 1504 snap 370 seq_num 1504) dirty exists 2014-07-01 17:11:52,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 3 to mira0435601-14 2014-07-01 17:11:52,952.952 INFO:teuthology.task.rados.rados.0.mira043.stdout:3710: expect (ObjNum 1770 snap 0 seq_num 1770) 2014-07-01 17:11:53,134.134 INFO:teuthology.task.ceph.osd.5:Started 2014-07-01 17:11:53,135.135 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok dump_ops_in_flight' 2014-07-01 17:11:53,186.186 INFO:teuthology.task.ceph.osd.5.mira028.stdout:starting osd.5 at :/0 osd_data /var/lib/ceph/osd/ceph-5 /var/lib/ceph/osd/ceph-5/journal 2014-07-01 17:11:53,233.233 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.141132 7f33af211780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 17:11:53,249.249 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156044 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 11c0941/mira0437250-351/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,249.249 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156293 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2f358941/mira0437250-236/246//16 in index: (2) No such file or directory 2014-07-01 17:11:53,250.250 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156349 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2f358941/mira0437250-236/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,251.251 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156552 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ba00ac41/mira0437250-398/25a//16 in index: (2) No such file or directory 2014-07-01 17:11:53,251.251 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156606 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ba00ac41/mira0437250-398/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,251.251 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.156661 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find cba253c1/mira0437250-436/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,252.252 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.157533 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find fa101851/mira0437250-257/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.157816 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ccbcfd91/mira0437250-86/2b3//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.157875 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ccbcfd91/mira0437250-86/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.157928 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f31a8fd1/mira0437250-73/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158691 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b0eede24/mira0437250-346/233//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158748 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b0eede24/mira0437250-346/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,302.302 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158801 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1814f664/mira0437250-44/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,303.303 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158856 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c1163b64/mira0437250-271/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,303.303 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158910 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find aead4a4/mira0437250-442/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,303.303 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.158962 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find bbbc79a4/mira0437250-462/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,303.303 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159156 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4d712ba4/mira0437250-107/21a//16 in index: (2) No such file or directory 2014-07-01 17:11:53,303.303 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159211 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4d712ba4/mira0437250-107/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159434 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 194b6fa4/mira0437250-219/289//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159491 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 194b6fa4/mira0437250-219/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159725 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b9bbc2e4/mira0437250-146/260//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159782 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b9bbc2e4/mira0437250-146/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.159838 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d7236ae4/mira0437250-229/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.160443 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 83336f3e/mira0437250-440/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.160648 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 628bc3be/mira0437250-475/246//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.160703 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 628bc3be/mira0437250-475/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.161284 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find fd6d3027/mira0437250-93/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.161497 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b4be08a7/mira0437250-112/250//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.161564 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b4be08a7/mira0437250-112/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,392.392 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162237 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find accf601f/mira0437250-108/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,393.393 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162351 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9e41571f/mira0437250-85/23d//16 in index: (2) No such file or directory 2014-07-01 17:11:53,393.393 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162377 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9e41571f/mira0437250-85/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,393.393 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162482 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find baaad39f/mira0437250-373/253//16 in index: (2) No such file or directory 2014-07-01 17:11:53,400.400 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162508 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find baaad39f/mira0437250-373/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162603 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e1d8849f/mira0437250-52/233//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162628 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e1d8849f/mira0437250-52/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162769 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9fa33d9f/mira0437250-293/2b3//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162794 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9fa33d9f/mira0437250-293/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162901 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1fe348df/mira0437250-92/273//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.162925 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1fe348df/mira0437250-92/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.163024 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 6e3fa9df/mira0437250-383/250//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.163050 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 6e3fa9df/mira0437250-383/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.163984 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 6830fb87/gc.1/head//9 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164010 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 685c6f7/gc.15/head//9 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164033 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 170a8acf/gc.6/head//9 in index: (2) No such file or directory 2014-07-01 17:11:53,401.401 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164056 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 61b8e3df/gc.8/head//9 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164607 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 31099063/notify.5/head//7 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164867 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find aaf7cb87/.dir.default.5268.25/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164895 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b4fcf4b7/.dir.default.5268.8/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164919 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1401dbe7/.dir.default.5268.28/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164943 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find daa920f7/.dir.default.5268.37/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,417.417 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164971 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1adafc0f/.dir.default.5268.30/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,418.418 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.164994 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f1e8eb3f/.dir.default.5268.24/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,419.419 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.165018 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c0cb0d3f/.dir.default.5268.62/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,419.419 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.165619 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b852b893/mds0_inotable/head//1 in index: (2) No such file or directory 2014-07-01 17:11:53,419.419 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.165853 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d90270ad/mds_snaptable/head//1 in index: (2) No such file or directory 2014-07-01 17:11:53,419.419 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166108 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c128e244/.bucket.meta.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:default.5268.27/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,420.420 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166145 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 38401274/.bucket.meta.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:default.5268.40/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,420.420 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166180 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c7838b74/.bucket.meta.0b2cf66925ad472ebd8cd8898ec7bc7d:default.5268.30/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,420.420 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166204 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 40bebfa4/.bucket.meta.鐉蔉댍ं嘅鐉鐊ः댍㜅툎댎鐊᠄蔉:default.5268.18/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,420.420 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166232 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find af83e3c4/.bucket.meta.ऀ嘆ं鐊嘅嘈㜄鐌嘈蔊鐊툏嘅댍:default.5268.38/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,421.421 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166254 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 8e5a94c4/.bucket.meta.6d37dc0c76344dd1803786c87148f2ed:default.5268.31/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,422.422 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166277 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ac091cf4/.bucket.meta.6b79dfc12e4d4941bdd77a3735a646cc:default.5268.1/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,422.422 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166300 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find a32af9ec/.bucket.meta.11e713e5b07645a6b091d7f6f2a7762d:default.5268.2/head//8 in index: (2) No such file or directory 2014-07-01 17:11:53,422.422 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166743 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 676be829/mira0437250-241/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,422.422 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166772 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c676fb29/mira0437250-258/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,422.422 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166875 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d7dd9f69/mira0437250-80/211//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.166963 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d7dd9f69/mira0437250-80/23d//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167050 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d7dd9f69/mira0437250-80/25f//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167075 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d7dd9f69/mira0437250-80/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167163 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4fe114a9/mira0437250-337/23d//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167187 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4fe114a9/mira0437250-337/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,423.423 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167279 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4f8d4ca9/mira0437250-66/253//16 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167309 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4f8d4ca9/mira0437250-66/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167332 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 909a67e9/mira0437250-196/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167441 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 33e1ce9/mira0437250-254/237//16 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167472 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 33e1ce9/mira0437250-254/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167683 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2a3ccc04/.dir.default.5268.4/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167707 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1d2487a4/.dir.default.5268.14/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,425.425 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167730 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9253c6b4/.dir.default.5268.29/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,426.426 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167758 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f9cb4fb4/.dir.default.5268.40/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,426.426 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167782 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 62ba044c/.dir.default.5268.18/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,426.426 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167807 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find ed01537c/.dir.default.5268.26/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,426.426 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167836 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2393bf8c/.dir.default.5268.9/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,426.426 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.167860 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 8b9c3dfc/.dir.default.5268.47/head//13 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.168066 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 79372b60/mira0435601-398/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.168093 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e2dd1a0/mira0435601-429/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.168775 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find c85f19a0/mira0435601-6/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.168805 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 70da5aa0/mira0435601-135/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.168830 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 3f28ee0/mira0435601-175/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,428.428 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169493 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 1b70422e/mira0437250-174/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,429.429 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169597 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2dc3006e/mira0437250-460/24d//16 in index: (2) No such file or directory 2014-07-01 17:11:53,429.429 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169693 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2dc3006e/mira0437250-460/273//16 in index: (2) No such file or directory 2014-07-01 17:11:53,429.429 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169718 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 2dc3006e/mira0437250-460/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,429.429 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169856 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d691476e/mira0437250-325/2a7//16 in index: (2) No such file or directory 2014-07-01 17:11:53,430.430 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169883 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find d691476e/mira0437250-325/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,430.430 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.169982 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e5753ae/mira0437250-123/22b//16 in index: (2) No such file or directory 2014-07-01 17:11:53,430.430 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170013 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e5753ae/mira0437250-123/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,431.431 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170312 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 8956c138/mira0435601-214/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:11:53,431.431 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170340 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find a689b638/mira0435601-1/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:11:53,431.431 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170365 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e0d4c0f8/mira0435601-213/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:11:53,431.431 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170390 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 7e816f8/mira0435601-161/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:11:53,431.431 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170422 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find e20346f8/mira0435601-287/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:11:53,432.432 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170628 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 107ff021/mira0435601-326/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,432.432 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170656 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 8de46561/mira0435601-389/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,432.432 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170686 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f2f47a61/mira0435601-56/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,433.433 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170716 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 4bae98e1/mira0435601-461/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,433.433 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.170741 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find f24bfee1/mira0435601-432/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:11:53,433.433 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171036 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 754b783d/mira0437250-106/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171164 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 5d05193d/mira0437250-79/27a//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171196 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 5d05193d/mira0437250-79/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171221 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 99ed1dbd/mira0437250-367/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171245 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find b017b7fd/mira0437250-450/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171738 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 38e7622b/mira0437250-479/221//16 in index: (2) No such file or directory 2014-07-01 17:11:53,434.434 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171764 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 38e7622b/mira0437250-479/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,435.435 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171951 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 30add42b/mira0437250-495/2a6//16 in index: (2) No such file or directory 2014-07-01 17:11:53,435.435 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.171978 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 30add42b/mira0437250-495/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,436.436 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.172104 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9d22ae6b/mira0437250-402/29e//16 in index: (2) No such file or directory 2014-07-01 17:11:53,436.436 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.172130 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 9d22ae6b/mira0437250-402/head//16 in index: (2) No such file or directory 2014-07-01 17:11:53,436.436 INFO:teuthology.task.ceph.osd.5.mira028.stderr:2014-07-01 08:11:53.172271 7f33af211780 -1 filestore(/var/lib/ceph/osd/ceph-5) could not find 47cfa0ab/mira0437250-366/2b3//16 in index: (2) No such file or directory 2014-07-01 17:11:53,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3253: done (2 left) 2014-07-01 17:11:53,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3254: done (1 left) 2014-07-01 17:11:53,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3255: done (0 left) 2014-07-01 17:11:53,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: read oid 134 snap -1 2014-07-01 17:11:53,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: read oid 274 snap -1 2014-07-01 17:11:53,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: snap_remove snap 346 2014-07-01 17:11:53,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: expect (ObjNum 665 snap 75 seq_num 665) 2014-07-01 17:11:53,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: expect (ObjNum 635 snap 60 seq_num 635) 2014-07-01 17:11:54,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3256: done (2 left) 2014-07-01 17:11:54,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3257: done (1 left) 2014-07-01 17:11:54,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3258: done (0 left) 2014-07-01 17:11:54,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3259: delete oid 319 current snap is 371 2014-07-01 17:11:54,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3259: done (0 left) 2014-07-01 17:11:54,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: snap_remove snap 369 2014-07-01 17:11:55,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3260: done (0 left) 2014-07-01 17:11:55,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: write oid 180 current snap is 371 2014-07-01 17:11:55,849.849 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: seq_num 1506 ranges {543411=1} 2014-07-01 17:11:55,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: writing mira0437250-180 from 543411 to 543412 tid 1 2014-07-01 17:11:55,850.850 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: read oid 379 snap 315 2014-07-01 17:11:55,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:3263: rollback oid 290 current snap is 371 2014-07-01 17:11:55,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 290 to 272 2014-07-01 17:11:55,852.852 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: snap_create 2014-07-01 17:11:56,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 1252 (ObjNum 289 snap 0 seq_num 289) dirty exists 2014-07-01 17:11:56,014.014 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 1 to mira0437250-180 2014-07-01 17:11:56,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 3 to mira0437250-180 2014-07-01 17:11:56,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: finishing write tid 4 to mira0437250-180 2014-07-01 17:11:56,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 1552 (ObjNum 1506 snap 371 seq_num 1506) dirty exists 2014-07-01 17:11:56,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 1 to mira0435601-389 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 1 to mira0435601-189 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 2 to mira0435601-189 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 4 to mira0435601-14 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 171 (ObjNum 2275 snap 0 seq_num 2275) dirty exists 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 2 to mira0435601-389 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: done (11 left) 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: done (10 left) 2014-07-01 17:11:56,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3707: done (9 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3708: done (8 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: done (7 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3710: done (6 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: done (5 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: done (4 left) 2014-07-01 17:11:56,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3715: done (3 left) 2014-07-01 17:11:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:3716: read oid 223 snap -1 2014-07-01 17:11:56,715.715 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 3 to mira0435601-389 2014-07-01 17:11:56,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 4 to mira0435601-389 2014-07-01 17:11:56,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 6 to mira0435601-389 2014-07-01 17:11:56,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: finishing write tid 7 to mira0435601-389 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 389 v 143 (ObjNum 2274 snap 0 seq_num 2274) dirty exists 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 3 to mira0435601-189 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 5 to mira0435601-189 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: done (3 left) 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 434 current snap is 0 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: seq_num 2278 ranges {180224=8192} 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: writing mira0435601-434 from 180224 to 188416 tid 1 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: delete oid 288 current snap is 0 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: finishing write tid 1 to mira0435601-444 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: finishing write tid 2 to mira0435601-444 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 6 to mira0435601-189 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 280 (ObjNum 2276 snap 0 seq_num 2276) dirty exists 2014-07-01 17:11:56,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: finishing write tid 3 to mira0435601-444 2014-07-01 17:11:56,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: finishing write tid 5 to mira0435601-444 2014-07-01 17:11:56,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: finishing write tid 6 to mira0435601-444 2014-07-01 17:11:56,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 251 (ObjNum 2277 snap 0 seq_num 2277) dirty exists 2014-07-01 17:11:56,884.884 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: finishing write tid 1 to mira0435601-434 2014-07-01 17:11:56,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: finishing write tid 3 to mira0435601-434 2014-07-01 17:11:56,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: done (4 left) 2014-07-01 17:11:56,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: done (3 left) 2014-07-01 17:11:56,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: done (2 left) 2014-07-01 17:11:56,885.885 INFO:teuthology.task.rados.rados.0.mira043.stdout:3719: read oid 363 snap -1 2014-07-01 17:11:56,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 84 current snap is 0 2014-07-01 17:11:56,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: seq_num 2280 ranges {4898816=61440} 2014-07-01 17:11:56,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: writing mira0435601-84 from 4898816 to 4960256 tid 1 2014-07-01 17:11:56,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: read oid 172 snap -1 2014-07-01 17:11:56,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: finishing write tid 4 to mira0435601-434 2014-07-01 17:11:56,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 434 v 253 (ObjNum 2278 snap 0 seq_num 2278) dirty exists 2014-07-01 17:11:56,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:3716: expect (ObjNum 2079 snap 0 seq_num 2079) 2014-07-01 17:11:56,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:3261: done (3 left) 2014-07-01 17:11:56,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:3262: done (2 left) 2014-07-01 17:11:56,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:3263: done (1 left) 2014-07-01 17:11:56,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3264: done (0 left) 2014-07-01 17:11:56,889.889 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: snap_remove snap 348 2014-07-01 17:11:57,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3265: done (0 left) 2014-07-01 17:11:57,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:3266: rollback oid 145 current snap is 372 2014-07-01 17:11:57,868.868 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 145 to 344 2014-07-01 17:11:57,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:3267: read oid 32 snap 344 2014-07-01 17:11:57,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:3268: rollback oid 404 current snap is 372 2014-07-01 17:11:57,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 404 to 279 2014-07-01 17:11:57,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: rollback oid 183 current snap is 372 2014-07-01 17:11:57,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 183 to 326 2014-07-01 17:11:57,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: snap_create 2014-07-01 17:11:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 615 (ObjNum 960 snap 175 seq_num 960) dirty exists 2014-07-01 17:11:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 944 (ObjNum 1227 snap 264 seq_num 1227) dirty exists 2014-07-01 17:11:58,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 999 (ObjNum 1338 snap 309 seq_num 1338) dirty exists 2014-07-01 17:11:58,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3719: expect (ObjNum 1952 snap 0 seq_num 1952) 2014-07-01 17:11:58,865.865 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3] out_osds: [5] dead_osds: [] live_osds: [1, 0, 2, 4, 3, 5] 2014-07-01 17:11:58,865.865 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:11:58,865.865 INFO:teuthology.task.thrashosds.thrasher:Adding osd 5 2014-07-01 17:11:58,865.865 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 5' 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3266: done (4 left) 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3267: done (3 left) 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3268: done (2 left) 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3269: done (1 left) 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3270: done (0 left) 2014-07-01 17:11:58,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:3271: snap_remove snap 291 2014-07-01 17:11:59,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3271: done (0 left) 2014-07-01 17:11:59,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: write oid 39 current snap is 373 2014-07-01 17:11:59,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: seq_num 1507 ranges {488242=753208,1288241=1} 2014-07-01 17:11:59,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: writing mira0437250-39 from 488242 to 1241450 tid 1 2014-07-01 17:11:59,906.906 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: writing mira0437250-39 from 1288241 to 1288242 tid 2 2014-07-01 17:11:59,907.907 INFO:teuthology.task.rados.rados.0.mira043.stdout:3273: snap_create 2014-07-01 17:11:59,914.914 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.5. 2014-07-01 17:11:59,924.924 INFO:teuthology.task.thrashosds.thrasher:Added osd 5 2014-07-01 17:12:00,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 1 to mira0437250-39 2014-07-01 17:12:00,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 2 to mira0437250-39 2014-07-01 17:12:00,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 4 to mira0437250-39 2014-07-01 17:12:00,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: finishing write tid 5 to mira0437250-39 2014-07-01 17:12:00,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 39 v 1078 (ObjNum 1507 snap 373 seq_num 1507) dirty exists 2014-07-01 17:12:00,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: finishing write tid 1 to mira0435601-84 2014-07-01 17:12:00,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: finishing write tid 3 to mira0435601-84 2014-07-01 17:12:00,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: finishing write tid 4 to mira0435601-84 2014-07-01 17:12:00,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 354 (ObjNum 2280 snap 0 seq_num 2280) dirty exists 2014-07-01 17:12:00,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3716: done (4 left) 2014-07-01 17:12:00,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: done (3 left) 2014-07-01 17:12:00,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3719: done (2 left) 2014-07-01 17:12:00,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: done (1 left) 2014-07-01 17:12:00,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: done (0 left) 2014-07-01 17:12:00,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3722: delete oid 424 current snap is 0 2014-07-01 17:12:00,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3272: done (1 left) 2014-07-01 17:12:00,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3273: done (0 left) 2014-07-01 17:12:00,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: delete oid 470 current snap is 374 2014-07-01 17:12:00,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3274: done (0 left) 2014-07-01 17:12:00,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3275: snap_remove snap 370 2014-07-01 17:12:00,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3722: done (0 left) 2014-07-01 17:12:00,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 397 current snap is 0 2014-07-01 17:12:00,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: seq_num 2282 ranges {6475776=659456} 2014-07-01 17:12:00,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: writing mira0435601-397 from 6475776 to 7135232 tid 1 2014-07-01 17:12:00,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 131 current snap is 0 2014-07-01 17:12:00,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: seq_num 2283 ranges {2392064=528384} 2014-07-01 17:12:00,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: writing mira0435601-131 from 2392064 to 2920448 tid 1 2014-07-01 17:12:01,057.057 INFO:teuthology.task.rados.rados.0.mira043.stdout:3725: read oid 342 snap -1 2014-07-01 17:12:01,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3726: read oid 5 snap -1 2014-07-01 17:12:01,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3727: delete oid 229 current snap is 0 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3725: done (4 left) 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3726: done (3 left) 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3727: done (2 left) 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3728: read oid 334 snap -1 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 474 current snap is 0 2014-07-01 17:12:01,126.126 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: seq_num 2285 ranges {4636672=430080,5066752=786432,5853184=417792} 2014-07-01 17:12:01,166.166 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: writing mira0435601-474 from 4636672 to 5066752 tid 1 2014-07-01 17:12:01,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: writing mira0435601-474 from 5066752 to 5853184 tid 2 2014-07-01 17:12:01,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: writing mira0435601-474 from 5853184 to 6270976 tid 3 2014-07-01 17:12:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: read oid 79 snap -1 2014-07-01 17:12:01,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: finishing write tid 1 to mira0435601-131 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: finishing write tid 3 to mira0435601-131 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: finishing write tid 4 to mira0435601-131 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 131 v 297 (ObjNum 2283 snap 0 seq_num 2283) dirty exists 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: done (4 left) 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 270 current snap is 0 2014-07-01 17:12:01,222.222 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: seq_num 2286 ranges {0=749568,749568=614400,1363968=573440,1937408=417792} 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: writing mira0435601-270 from 0 to 749568 tid 1 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: writing mira0435601-270 from 749568 to 1363968 tid 2 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: writing mira0435601-270 from 1363968 to 1937408 tid 3 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: writing mira0435601-270 from 1937408 to 2355200 tid 4 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3732: read oid 244 snap -1 2014-07-01 17:12:01,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: expect (ObjNum 1242 snap 0 seq_num 1242) 2014-07-01 17:12:01,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3275: done (0 left) 2014-07-01 17:12:01,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: snap_remove snap 279 2014-07-01 17:12:02,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:3276: done (0 left) 2014-07-01 17:12:02,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: snap_remove snap 354 2014-07-01 17:12:02,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3728: expect (ObjNum 532 snap 0 seq_num 532) 2014-07-01 17:12:03,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:3277: done (0 left) 2014-07-01 17:12:03,537.537 INFO:teuthology.task.rados.rados.0.mira043.stdout:3278: snap_create 2014-07-01 17:12:04,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3732: expect (ObjNum 1982 snap 0 seq_num 1982) 2014-07-01 17:12:04,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3278: done (0 left) 2014-07-01 17:12:04,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: write oid 458 current snap is 375 2014-07-01 17:12:04,542.542 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: seq_num 1509 ranges {527277=698865,1727276=1} 2014-07-01 17:12:04,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: writing mira0437250-458 from 527277 to 1226142 tid 1 2014-07-01 17:12:04,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: writing mira0437250-458 from 1727276 to 1727277 tid 2 2014-07-01 17:12:04,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: snap_create 2014-07-01 17:12:04,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 1 to mira0437250-458 2014-07-01 17:12:04,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 2 to mira0437250-458 2014-07-01 17:12:04,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 4 to mira0437250-458 2014-07-01 17:12:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: finishing write tid 5 to mira0437250-458 2014-07-01 17:12:04,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 1153 (ObjNum 1509 snap 375 seq_num 1509) dirty exists 2014-07-01 17:12:04,924.924 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 3, 5] 2014-07-01 17:12:04,925.925 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:12:04,925.925 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:12:04,926.926 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:12:05,548.548 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:12:05,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3279: done (1 left) 2014-07-01 17:12:05,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3280: done (0 left) 2014-07-01 17:12:05,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: write oid 370 current snap is 376 2014-07-01 17:12:05,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: seq_num 1510 ranges {471083=422094,1572552=506267,2471082=1} 2014-07-01 17:12:05,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0437250-370 from 471083 to 893177 tid 1 2014-07-01 17:12:05,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0437250-370 from 1572552 to 2078819 tid 2 2014-07-01 17:12:05,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: writing mira0437250-370 from 2471082 to 2471083 tid 3 2014-07-01 17:12:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: delete oid 404 current snap is 376 2014-07-01 17:12:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 1 to mira0437250-370 2014-07-01 17:12:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 2 to mira0437250-370 2014-07-01 17:12:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 3 to mira0437250-370 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 5 to mira0437250-370 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3282: done (1 left) 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: write oid 249 current snap is 376 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: seq_num 1512 ranges {628518=401798,1518191=797057,2967797=410500,3428517=1} 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: writing mira0437250-249 from 628518 to 1030316 tid 1 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: writing mira0437250-249 from 1518191 to 2315248 tid 2 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: writing mira0437250-249 from 2967797 to 3378297 tid 3 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: writing mira0437250-249 from 3428517 to 3428518 tid 4 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: rollback oid 27 current snap is 376 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 27 to 330 2014-07-01 17:12:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: finishing write tid 6 to mira0437250-370 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 370 v 1422 (ObjNum 1510 snap 376 seq_num 1510) dirty exists 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3281: done (2 left) 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: read oid 347 snap -1 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: write oid 88 current snap is 376 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: seq_num 1513 ranges {172323=1} 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: writing mira0437250-88 from 172323 to 172324 tid 1 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: write oid 73 current snap is 376 2014-07-01 17:12:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: seq_num 1514 ranges {517154=400000} 2014-07-01 17:12:05,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: writing mira0437250-73 from 517154 to 917154 tid 1 2014-07-01 17:12:05,742.742 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: snap_remove snap 352 2014-07-01 17:12:05,853.853 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: expect (ObjNum 1221 snap 261 seq_num 1221) 2014-07-01 17:12:06,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 1 to mira0437250-249 2014-07-01 17:12:06,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 2 to mira0437250-249 2014-07-01 17:12:06,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 3 to mira0437250-249 2014-07-01 17:12:06,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 1 to mira0437250-88 2014-07-01 17:12:06,303.303 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 3 to mira0437250-88 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 4 to mira0437250-249 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 6 to mira0437250-249 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: finishing write tid 7 to mira0437250-249 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 858 (ObjNum 1512 snap 376 seq_num 1512) dirty exists 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: finishing write tid 4 to mira0437250-88 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 88 v 1614 (ObjNum 1513 snap 376 seq_num 1513) dirty exists 2014-07-01 17:12:06,304.304 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 27 v 1654 (ObjNum 1296 snap 293 seq_num 1296) dirty exists 2014-07-01 17:12:06,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3283: done (5 left) 2014-07-01 17:12:06,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3284: done (4 left) 2014-07-01 17:12:06,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3285: done (3 left) 2014-07-01 17:12:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3286: done (2 left) 2014-07-01 17:12:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3288: done (1 left) 2014-07-01 17:12:06,820.820 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: read oid 383 snap -1 2014-07-01 17:12:06,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: snap_create 2014-07-01 17:12:06,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 1 to mira0437250-73 2014-07-01 17:12:06,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 3 to mira0437250-73 2014-07-01 17:12:06,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: finishing write tid 4 to mira0437250-73 2014-07-01 17:12:06,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 73 v 593 (ObjNum 1514 snap 376 seq_num 1514) dirty exists 2014-07-01 17:12:07,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3287: done (2 left) 2014-07-01 17:12:07,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3290: done (1 left) 2014-07-01 17:12:07,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: read oid 81 snap -1 2014-07-01 17:12:07,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3292: snap_create 2014-07-01 17:12:07,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: expect (ObjNum 689 snap 85 seq_num 689) 2014-07-01 17:12:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: finishing write tid 1 to mira0435601-397 2014-07-01 17:12:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: finishing write tid 3 to mira0435601-397 2014-07-01 17:12:07,874.874 INFO:teuthology.task.rados.rados.0.mira043.stdout:3728: done (5 left) 2014-07-01 17:12:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: done (4 left) 2014-07-01 17:12:07,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:3732: done (3 left) 2014-07-01 17:12:07,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: read oid 334 snap -1 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: read oid 56 snap -1 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 1 to mira0435601-270 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: finishing write tid 4 to mira0435601-397 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 171 (ObjNum 2282 snap 0 seq_num 2282) dirty exists 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 2 to mira0435601-270 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 3 to mira0435601-270 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 4 to mira0435601-270 2014-07-01 17:12:07,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 6 to mira0435601-270 2014-07-01 17:12:07,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: finishing write tid 7 to mira0435601-270 2014-07-01 17:12:07,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 270 v 178 (ObjNum 2286 snap 0 seq_num 2286) dirty exists 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: finishing write tid 1 to mira0435601-474 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: finishing write tid 2 to mira0435601-474 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: done (4 left) 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: done (3 left) 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 219 current snap is 0 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: seq_num 2287 ranges {1310720=778240,2088960=401408} 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: writing mira0435601-219 from 1310720 to 2088960 tid 1 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: writing mira0435601-219 from 2088960 to 2490368 tid 2 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: read oid 163 snap -1 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: finishing write tid 3 to mira0435601-474 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: finishing write tid 5 to mira0435601-474 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: finishing write tid 6 to mira0435601-474 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 474 v 230 (ObjNum 2285 snap 0 seq_num 2285) dirty exists 2014-07-01 17:12:08,044.044 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: done (4 left) 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: seq_num 2288 ranges {6377472=651264,7028736=401408} 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: writing mira0435601-351 from 6377472 to 7028736 tid 1 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: writing mira0435601-351 from 7028736 to 7430144 tid 2 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3738: delete oid 360 current snap is 0 2014-07-01 17:12:08,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: expect (ObjNum 2119 snap 0 seq_num 2119) 2014-07-01 17:12:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3291: done (2 left) 2014-07-01 17:12:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3292: done (1 left) 2014-07-01 17:12:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: write oid 184 current snap is 378 2014-07-01 17:12:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: seq_num 1515 ranges {728735=400000} 2014-07-01 17:12:08,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: writing mira0437250-184 from 728735 to 1128735 tid 1 2014-07-01 17:12:08,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: write oid 55 current snap is 378 2014-07-01 17:12:08,581.581 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: seq_num 1516 ranges {673565=476536,1635776=237789} 2014-07-01 17:12:08,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: writing mira0437250-55 from 673565 to 1150101 tid 1 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: writing mira0437250-55 from 1635776 to 1873565 tid 2 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3295: delete oid 272 current snap is 378 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 1 to mira0437250-184 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 3 to mira0437250-184 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: finishing write tid 4 to mira0437250-184 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 1343 (ObjNum 1515 snap 378 seq_num 1515) dirty exists 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3293: done (3 left) 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3295: done (2 left) 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: read oid 400 snap 376 2014-07-01 17:12:08,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3297: snap_remove snap 363 2014-07-01 17:12:09,149.149 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:12:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3296: done (3 left) 2014-07-01 17:12:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3297: done (2 left) 2014-07-01 17:12:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3298: rollback oid 295 current snap is 378 2014-07-01 17:12:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 295 to 347 2014-07-01 17:12:09,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: write oid 267 current snap is 378 2014-07-01 17:12:09,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: seq_num 1518 ranges {712601=571568,1977320=335281} 2014-07-01 17:12:09,596.596 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: writing mira0437250-267 from 712601 to 1284169 tid 1 2014-07-01 17:12:09,611.611 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: writing mira0437250-267 from 1977320 to 2312601 tid 2 2014-07-01 17:12:09,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3300: read oid 335 snap -1 2014-07-01 17:12:09,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: snap_remove snap 364 2014-07-01 17:12:09,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3300: expect (ObjNum 1371 snap 311 seq_num 1371) 2014-07-01 17:12:09,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: expect (ObjNum 2205 snap 0 seq_num 2205) 2014-07-01 17:12:10,631.631 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:12:10,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 1 to mira0437250-267 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 2 to mira0437250-267 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 4 to mira0437250-267 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: finishing write tid 5 to mira0437250-267 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 1212 (ObjNum 1518 snap 378 seq_num 1518) dirty exists 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 1 to mira0437250-55 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 2 to mira0437250-55 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 4 to mira0437250-55 2014-07-01 17:12:10,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3299: done (5 left) 2014-07-01 17:12:10,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3300: done (4 left) 2014-07-01 17:12:10,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3301: done (3 left) 2014-07-01 17:12:10,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: read oid 312 snap -1 2014-07-01 17:12:10,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: write oid 96 current snap is 378 2014-07-01 17:12:10,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: seq_num 1519 ranges {657430=676621,1789628=674919,3057429=1} 2014-07-01 17:12:10,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0437250-96 from 657430 to 1334051 tid 1 2014-07-01 17:12:10,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0437250-96 from 1789628 to 2464547 tid 2 2014-07-01 17:12:10,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: writing mira0437250-96 from 3057429 to 3057430 tid 3 2014-07-01 17:12:10,821.821 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: write oid 102 current snap is 378 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: seq_num 1520 ranges {469012=799849,2019089=496538,2922305=346707} 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: writing mira0437250-102 from 469012 to 1268861 tid 1 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: writing mira0437250-102 from 2019089 to 2515627 tid 2 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: writing mira0437250-102 from 2922305 to 3269012 tid 3 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3305: snap_remove snap 349 2014-07-01 17:12:10,822.822 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: expect (ObjNum 1484 snap 363 seq_num 1484) 2014-07-01 17:12:11,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 1 to mira0437250-96 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 2 to mira0437250-96 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 3 to mira0437250-96 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 5 to mira0437250-96 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: finishing write tid 6 to mira0437250-96 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 96 v 960 (ObjNum 1519 snap 378 seq_num 1519) dirty exists 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 1682 (ObjNum 626 snap 53 seq_num 626) dirty exists 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: finishing write tid 5 to mira0437250-55 2014-07-01 17:12:11,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 1725 (ObjNum 1516 snap 378 seq_num 1516) dirty exists 2014-07-01 17:12:11,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3294: done (6 left) 2014-07-01 17:12:11,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3298: done (5 left) 2014-07-01 17:12:11,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3302: done (4 left) 2014-07-01 17:12:11,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3303: done (3 left) 2014-07-01 17:12:11,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3305: done (2 left) 2014-07-01 17:12:11,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: delete oid 358 current snap is 378 2014-07-01 17:12:11,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3306: done (2 left) 2014-07-01 17:12:11,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: write oid 469 current snap is 378 2014-07-01 17:12:11,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: seq_num 1522 ranges {758671=1} 2014-07-01 17:12:11,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: writing mira0437250-469 from 758671 to 758672 tid 1 2014-07-01 17:12:11,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3308: snap_create 2014-07-01 17:12:12,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 1 to mira0437250-469 2014-07-01 17:12:12,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 3 to mira0437250-469 2014-07-01 17:12:12,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: finishing write tid 4 to mira0437250-469 2014-07-01 17:12:12,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 1625 (ObjNum 1522 snap 378 seq_num 1522) dirty exists 2014-07-01 17:12:12,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3307: done (3 left) 2014-07-01 17:12:12,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3308: done (2 left) 2014-07-01 17:12:12,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: read oid 398 snap -1 2014-07-01 17:12:12,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3310: snap_create 2014-07-01 17:12:12,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: expect (ObjNum 1369 snap 311 seq_num 1369) 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: finishing write tid 1 to mira0435601-219 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: finishing write tid 2 to mira0435601-219 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: finishing write tid 4 to mira0435601-219 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: finishing write tid 5 to mira0435601-219 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 219 v 379 (ObjNum 2287 snap 0 seq_num 2287) dirty exists 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: finishing write tid 1 to mira0435601-351 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: finishing write tid 2 to mira0435601-351 2014-07-01 17:12:12,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: finishing write tid 4 to mira0435601-351 2014-07-01 17:12:12,867.867 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: expect (ObjNum 532 snap 0 seq_num 532) 2014-07-01 17:12:13,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 1 to mira0437250-102 2014-07-01 17:12:13,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 2 to mira0437250-102 2014-07-01 17:12:13,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 3 to mira0437250-102 2014-07-01 17:12:13,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 5 to mira0437250-102 2014-07-01 17:12:13,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: finishing write tid 6 to mira0437250-102 2014-07-01 17:12:13,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 1302 (ObjNum 1520 snap 378 seq_num 1520) dirty exists 2014-07-01 17:12:13,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: expect (ObjNum 1333 snap 308 seq_num 1333) 2014-07-01 17:12:14,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: finishing write tid 5 to mira0435601-351 2014-07-01 17:12:14,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 403 (ObjNum 2288 snap 0 seq_num 2288) dirty exists 2014-07-01 17:12:14,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: done (5 left) 2014-07-01 17:12:14,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: done (4 left) 2014-07-01 17:12:14,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: done (3 left) 2014-07-01 17:12:14,653.653 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: done (2 left) 2014-07-01 17:12:14,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: done (1 left) 2014-07-01 17:12:14,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3738: done (0 left) 2014-07-01 17:12:14,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 17:12:14,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: seq_num 2290 ranges {1912832=548864,2461696=593920,3055616=708608,3764224=303104} 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: writing mira0435601-186 from 1912832 to 2461696 tid 1 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: writing mira0435601-186 from 2461696 to 3055616 tid 2 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: writing mira0435601-186 from 3055616 to 3764224 tid 3 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: writing mira0435601-186 from 3764224 to 4067328 tid 4 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3740: read oid 85 snap -1 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 157 current snap is 0 2014-07-01 17:12:14,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: seq_num 2291 ranges {6041600=466944} 2014-07-01 17:12:14,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: writing mira0435601-157 from 6041600 to 6508544 tid 1 2014-07-01 17:12:14,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: read oid 21 snap -1 2014-07-01 17:12:14,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:3740: done (3 left) 2014-07-01 17:12:14,860.860 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 17:12:14,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: seq_num 2292 ranges {9310208=339968} 2014-07-01 17:12:14,968.968 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: writing mira0435601-311 from 9310208 to 9650176 tid 1 2014-07-01 17:12:14,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:3744: read oid 438 snap -1 2014-07-01 17:12:14,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: read oid 409 snap -1 2014-07-01 17:12:14,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 331 current snap is 0 2014-07-01 17:12:14,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: seq_num 2293 ranges {581632=770048,1351680=589824,1941504=212992} 2014-07-01 17:12:15,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: writing mira0435601-331 from 581632 to 1351680 tid 1 2014-07-01 17:12:15,042.042 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: writing mira0435601-331 from 1351680 to 1941504 tid 2 2014-07-01 17:12:15,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: writing mira0435601-331 from 1941504 to 2154496 tid 3 2014-07-01 17:12:15,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 198 current snap is 0 2014-07-01 17:12:15,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: seq_num 2294 ranges {2068480=393216} 2014-07-01 17:12:15,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: writing mira0435601-198 from 2068480 to 2461696 tid 1 2014-07-01 17:12:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3748: read oid 188 snap -1 2014-07-01 17:12:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3744: done (8 left) 2014-07-01 17:12:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 231 current snap is 0 2014-07-01 17:12:15,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: seq_num 2295 ranges {2576384=561152,3137536=405504,3543040=585728,4128768=409600,4538368=204800} 2014-07-01 17:12:15,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: writing mira0435601-231 from 2576384 to 3137536 tid 1 2014-07-01 17:12:15,186.186 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: writing mira0435601-231 from 3137536 to 3543040 tid 2 2014-07-01 17:12:15,205.205 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: writing mira0435601-231 from 3543040 to 4128768 tid 3 2014-07-01 17:12:15,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: writing mira0435601-231 from 4128768 to 4538368 tid 4 2014-07-01 17:12:15,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: writing mira0435601-231 from 4538368 to 4743168 tid 5 2014-07-01 17:12:15,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 493 current snap is 0 2014-07-01 17:12:15,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: seq_num 2296 ranges {1011712=589824,1601536=401408} 2014-07-01 17:12:15,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: writing mira0435601-493 from 1011712 to 1601536 tid 1 2014-07-01 17:12:15,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: writing mira0435601-493 from 1601536 to 2002944 tid 2 2014-07-01 17:12:15,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 238 current snap is 0 2014-07-01 17:12:15,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: seq_num 2297 ranges {0=614400,614400=413696,1028096=532480,1560576=573440,2134016=86016} 2014-07-01 17:12:15,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: writing mira0435601-238 from 0 to 614400 tid 1 2014-07-01 17:12:15,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: writing mira0435601-238 from 614400 to 1028096 tid 2 2014-07-01 17:12:15,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: writing mira0435601-238 from 1028096 to 1560576 tid 3 2014-07-01 17:12:15,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: writing mira0435601-238 from 1560576 to 2134016 tid 4 2014-07-01 17:12:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: writing mira0435601-238 from 2134016 to 2220032 tid 5 2014-07-01 17:12:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: read oid 179 snap -1 2014-07-01 17:12:15,462.462 INFO:teuthology.task.rados.rados.0.mira043.stdout:3748: expect (ObjNum 1741 snap 0 seq_num 1741) 2014-07-01 17:12:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3289: done (3 left) 2014-07-01 17:12:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3304: done (2 left) 2014-07-01 17:12:15,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3309: done (1 left) 2014-07-01 17:12:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:3310: done (0 left) 2014-07-01 17:12:15,576.576 INFO:teuthology.task.rados.rados.0.mira043.stdout:3311: snap_remove snap 378 2014-07-01 17:12:15,640.640 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [1, 0, 2, 4, 3, 5] 2014-07-01 17:12:15,641.641 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:12:15,641.641 INFO:teuthology.task.thrashosds.thrasher:Killing osd 1, live_osds are [1, 0, 2, 4, 3, 5] 2014-07-01 17:12:15,641.641 DEBUG:teuthology.task.ceph.osd.1:waiting for process to exit 2014-07-01 17:12:16,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3311: done (0 left) 2014-07-01 17:12:16,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: rollback oid 92 current snap is 380 2014-07-01 17:12:16,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 92 to 347 2014-07-01 17:12:16,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3313: snap_create 2014-07-01 17:12:17,758.758 INFO:teuthology.task.rados.rados.0.mira043.stdout:3313: done (1 left) 2014-07-01 17:12:17,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: read oid 282 snap -1 2014-07-01 17:12:17,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:3315: delete oid 178 current snap is 381 2014-07-01 17:12:17,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: expect (ObjNum 1504 snap 370 seq_num 1504) 2014-07-01 17:12:18,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: expect (ObjNum 1999 snap 0 seq_num 1999) 2014-07-01 17:12:21,642.642 INFO:teuthology.task.ceph.osd.1:Stopped 2014-07-01 17:12:22,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: finishing write tid 1 to mira0435601-157 2014-07-01 17:12:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: finishing write tid 3 to mira0435601-157 2014-07-01 17:12:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 1 to mira0435601-231 2014-07-01 17:12:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 2 to mira0435601-231 2014-07-01 17:12:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 3 to mira0435601-231 2014-07-01 17:12:22,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 1 to mira0435601-186 2014-07-01 17:12:22,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 4 to mira0435601-231 2014-07-01 17:12:22,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 5 to mira0435601-231 2014-07-01 17:12:22,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 7 to mira0435601-231 2014-07-01 17:12:22,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 2 to mira0435601-186 2014-07-01 17:12:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 3 to mira0435601-186 2014-07-01 17:12:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3748: done (11 left) 2014-07-01 17:12:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: done (10 left) 2014-07-01 17:12:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3753: read oid 226 snap -1 2014-07-01 17:12:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 4 to mira0435601-186 2014-07-01 17:12:22,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 6 to mira0435601-186 2014-07-01 17:12:22,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: finishing write tid 1 to mira0435601-331 2014-07-01 17:12:22,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: finishing write tid 2 to mira0435601-331 2014-07-01 17:12:22,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: finishing write tid 3 to mira0435601-331 2014-07-01 17:12:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: finishing write tid 5 to mira0435601-331 2014-07-01 17:12:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: finishing write tid 1 to mira0435601-493 2014-07-01 17:12:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: finishing write tid 2 to mira0435601-493 2014-07-01 17:12:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: finishing write tid 4 to mira0435601-493 2014-07-01 17:12:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: finishing write tid 7 to mira0435601-186 2014-07-01 17:12:22,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 215 (ObjNum 2290 snap 0 seq_num 2290) dirty exists 2014-07-01 17:12:22,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: finishing write tid 4 to mira0435601-157 2014-07-01 17:12:22,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 316 (ObjNum 2291 snap 0 seq_num 2291) dirty exists 2014-07-01 17:12:22,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 1 to mira0435601-238 2014-07-01 17:12:22,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: done (10 left) 2014-07-01 17:12:22,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: done (9 left) 2014-07-01 17:12:22,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 47 current snap is 0 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: seq_num 2298 ranges {0=733184} 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: writing mira0435601-47 from 0 to 733184 tid 1 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: seq_num 2299 ranges {262144=761856,1024000=630784,1654784=573440} 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: writing mira0435601-296 from 262144 to 1024000 tid 1 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: writing mira0435601-296 from 1024000 to 1654784 tid 2 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: writing mira0435601-296 from 1654784 to 2228224 tid 3 2014-07-01 17:12:22,521.521 INFO:teuthology.task.rados.rados.0.mira043.stdout:3756: read oid 337 snap -1 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: finishing write tid 1 to mira0435601-311 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: finishing write tid 3 to mira0435601-311 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 2 to mira0435601-238 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 3 to mira0435601-238 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: finishing write tid 4 to mira0435601-311 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 291 (ObjNum 2292 snap 0 seq_num 2292) dirty exists 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 4 to mira0435601-238 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 5 to mira0435601-238 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 7 to mira0435601-238 2014-07-01 17:12:22,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: expect (ObjNum 2038 snap 0 seq_num 2038) 2014-07-01 17:12:26,643.643 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [1] live_osds: [0, 2, 4, 3, 5] 2014-07-01 17:12:26,644.644 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:12:26,644.644 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:12:26,644.644 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:12:26,645.645 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:12:26,645.645 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:12:28,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: expect (ObjNum 2042 snap 0 seq_num 2042) 2014-07-01 17:12:32,108.108 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [1] live_osds: [0, 2, 4, 3, 5] 2014-07-01 17:12:32,109.109 INFO:teuthology.task.thrashosds.thrasher:Reviving osd 1 2014-07-01 17:12:32,109.109 INFO:teuthology.task.ceph.osd.1:Restarting daemon 2014-07-01 17:12:32,109.109 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2014-07-01 17:12:32,478.478 INFO:teuthology.task.ceph.osd.1:Started 2014-07-01 17:12:32,478.478 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok dump_ops_in_flight' 2014-07-01 17:12:32,626.626 INFO:teuthology.task.ceph.osd.1.mira079.stdout:starting osd.1 at :/0 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal 2014-07-01 17:12:32,676.676 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.588324 7fe6af591780 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-07-01 17:12:32,737.737 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.649390 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find fa101851/mira0437250-257/head//16 in index: (2) No such file or directory 2014-07-01 17:12:32,737.737 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.649560 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find ccbcfd91/mira0437250-86/2b3//16 in index: (2) No such file or directory 2014-07-01 17:12:32,738.738 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.649600 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find ccbcfd91/mira0437250-86/snapdir//16 in index: (2) No such file or directory 2014-07-01 17:12:32,738.738 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.649639 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find f31a8fd1/mira0437250-73/head//16 in index: (2) No such file or directory 2014-07-01 17:12:32,740.740 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.650004 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 627ffc0c/mira0435601-22/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:12:32,740.740 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.650032 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find cde298c/mira0435601-108/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:12:32,741.741 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.650056 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find e29223cc/mira0435601-379/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:12:32,811.811 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.650082 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 529e35cc/mira0435601-176/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:12:32,811.811 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.650106 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 7cecddcc/mira0435601-333/head//3/18446744073709551615/0 in index: (2) No such file or directory 2014-07-01 17:12:32,811.811 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652257 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 79372b60/mira0435601-398/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:12:32,811.811 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652292 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find e2dd1a0/mira0435601-429/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:12:32,811.811 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652317 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find c85f19a0/mira0435601-6/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:12:32,812.812 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652343 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 70da5aa0/mira0435601-135/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:12:32,812.812 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652368 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 3f28ee0/mira0435601-175/head//3/18446744073709551615/1 in index: (2) No such file or directory 2014-07-01 17:12:32,813.813 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652808 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find e0972a05/mira0437250-238/head//16 in index: (2) No such file or directory 2014-07-01 17:12:32,813.813 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652835 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find a090d785/mira0437250-158/head//16 in index: (2) No such file or directory 2014-07-01 17:12:32,813.813 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.652937 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 56f3985/mira0437250-1/253//16 in index: (2) No such file or directory 2014-07-01 17:12:32,813.813 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.653064 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 56f3985/mira0437250-1/2af//16 in index: (2) No such file or directory 2014-07-01 17:12:32,816.816 INFO:teuthology.task.ceph.osd.1.mira079.stderr:2014-07-01 08:12:32.653113 7fe6af591780 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 56f3985/mira0437250-1/head//16 in index: (2) No such file or directory 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:no valid command found; 10 closest matches: 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:config show 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:help 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:log dump 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:get_command_descriptions 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:git_version 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:config set [...] 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:version 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:2 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:config get 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:0 2014-07-01 17:12:32,892.892 INFO:teuthology.orchestra.run.mira079.stderr:admin_socket: invalid command 2014-07-01 17:12:32,897.897 INFO:teuthology.task.thrashosds.ceph_manager:waiting on admin_socket for 1, ['dump_ops_in_flight'] 2014-07-01 17:12:34,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 1 to mira0435601-198 2014-07-01 17:12:34,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 3 to mira0435601-198 2014-07-01 17:12:34,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 4 to mira0435601-198 2014-07-01 17:12:34,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 274 (ObjNum 2294 snap 0 seq_num 2294) dirty exists 2014-07-01 17:12:34,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: done (11 left) 2014-07-01 17:12:34,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: done (10 left) 2014-07-01 17:12:34,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: done (9 left) 2014-07-01 17:12:34,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: done (8 left) 2014-07-01 17:12:34,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 402 current snap is 0 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: seq_num 2300 ranges {3559424=434176,3993600=614400,4608000=589824} 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0435601-402 from 3559424 to 3993600 tid 1 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0435601-402 from 3993600 to 4608000 tid 2 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0435601-402 from 4608000 to 5197824 tid 3 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3758: delete oid 197 current snap is 0 2014-07-01 17:12:34,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3753: expect (ObjNum 2260 snap 0 seq_num 2260) 2014-07-01 17:12:34,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 92 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:12:37,898.898 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok dump_ops_in_flight' 2014-07-01 17:12:38,364.364 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 17:12:38,944.944 INFO:teuthology.task.rados.rados.0.mira043.stdout:3756: expect (ObjNum 2239 snap 0 seq_num 2239) 2014-07-01 17:12:39,589.589 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 17:12:39,597.597 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 17:12:39,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: finishing write tid 1 to mira0435601-47 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: finishing write tid 3 to mira0435601-47 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: finishing write tid 4 to mira0435601-47 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 47 v 284 (ObjNum 2298 snap 0 seq_num 2298) dirty exists 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: finishing write tid 8 to mira0435601-231 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 184 (ObjNum 2295 snap 0 seq_num 2295) dirty exists 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 1 to mira0435601-402 2014-07-01 17:12:39,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: finishing write tid 8 to mira0435601-238 2014-07-01 17:12:39,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 238 v 240 (ObjNum 2297 snap 0 seq_num 2297) dirty exists 2014-07-01 17:12:39,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 2 to mira0435601-402 2014-07-01 17:12:39,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 3 to mira0435601-402 2014-07-01 17:12:39,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 5 to mira0435601-402 2014-07-01 17:12:39,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 6 to mira0435601-402 2014-07-01 17:12:39,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 402 v 360 (ObjNum 2300 snap 0 seq_num 2300) dirty exists 2014-07-01 17:12:39,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: finishing write tid 1 to mira0435601-296 2014-07-01 17:12:39,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: finishing write tid 2 to mira0435601-296 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: finishing write tid 3 to mira0435601-296 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: finishing write tid 5 to mira0435601-296 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: finishing write tid 5 to mira0435601-493 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 493 v 302 (ObjNum 2296 snap 0 seq_num 2296) dirty exists 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: finishing write tid 6 to mira0435601-331 2014-07-01 17:12:39,696.696 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 299 (ObjNum 2293 snap 0 seq_num 2293) dirty exists 2014-07-01 17:12:39,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: finishing write tid 6 to mira0435601-296 2014-07-01 17:12:39,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 236 (ObjNum 2299 snap 0 seq_num 2299) dirty exists 2014-07-01 17:12:39,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: done (9 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: done (8 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: done (7 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: done (6 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3753: done (5 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: done (4 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: done (3 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3756: done (2 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: done (1 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3758: done (0 left) 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 254 current snap is 0 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: seq_num 2302 ranges {0=581632,581632=782336,1363968=20480} 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: writing mira0435601-254 from 0 to 581632 tid 1 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: writing mira0435601-254 from 581632 to 1363968 tid 2 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: writing mira0435601-254 from 1363968 to 1384448 tid 3 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 430 current snap is 0 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: seq_num 2303 ranges {4866048=204800} 2014-07-01 17:12:39,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: writing mira0435601-430 from 4866048 to 5070848 tid 1 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3761: read oid 306 snap -1 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: finishing write tid 1 to mira0435601-254 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: finishing write tid 2 to mira0435601-254 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: finishing write tid 3 to mira0435601-254 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3762: read oid 263 snap -1 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: finishing write tid 5 to mira0435601-254 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: finishing write tid 6 to mira0435601-254 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 254 v 244 (ObjNum 2302 snap 0 seq_num 2302) dirty exists 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: done (3 left) 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: read oid 496 snap -1 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3764: read oid 119 snap -1 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 147 current snap is 0 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: seq_num 2304 ranges {7532544=774144,8306688=598016,8904704=606208} 2014-07-01 17:12:39,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: writing mira0435601-147 from 7532544 to 8306688 tid 1 2014-07-01 17:12:39,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: writing mira0435601-147 from 8306688 to 8904704 tid 2 2014-07-01 17:12:39,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: writing mira0435601-147 from 8904704 to 9510912 tid 3 2014-07-01 17:12:39,883.883 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 48 current snap is 0 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: seq_num 2305 ranges {5971968=802816} 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: writing mira0435601-48 from 5971968 to 6774784 tid 1 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: read oid 206 snap -1 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 1 to mira0435601-430 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 3 to mira0435601-430 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 4 to mira0435601-430 2014-07-01 17:12:40,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 430 v 238 (ObjNum 2303 snap 0 seq_num 2303) dirty exists 2014-07-01 17:12:40,028.028 INFO:teuthology.task.rados.rados.0.mira043.stdout:3764: expect (ObjNum 1329 snap 0 seq_num 1329) 2014-07-01 17:12:40,606.606 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 17:12:40,628.628 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 17:12:41,638.638 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 17:12:41,650.650 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 17:12:42,640.640 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 17:12:42,653.653 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 17:12:42,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3761: expect (ObjNum 2270 snap 0 seq_num 2270) 2014-07-01 17:12:43,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:3762: expect (ObjNum 1991 snap 0 seq_num 1991) 2014-07-01 17:12:43,657.657 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 17:12:43,665.665 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 17:12:44,631.631 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 17:12:44,658.658 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 17:12:44,658.658 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:12:45,429.429 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:12:45,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: finishing write tid 1 to mira0435601-147 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: finishing write tid 2 to mira0435601-147 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: finishing write tid 3 to mira0435601-147 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: finishing write tid 5 to mira0435601-147 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: finishing write tid 6 to mira0435601-147 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 147 v 175 (ObjNum 2304 snap 0 seq_num 2304) dirty exists 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: done (7 left) 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3761: done (6 left) 2014-07-01 17:12:45,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3762: done (5 left) 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3764: done (4 left) 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: done (3 left) 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3768: delete oid 164 current snap is 0 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 1 to mira0435601-48 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 3 to mira0435601-48 2014-07-01 17:12:45,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 4 to mira0435601-48 2014-07-01 17:12:45,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 48 v 405 (ObjNum 2305 snap 0 seq_num 2305) dirty exists 2014-07-01 17:12:45,827.827 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: expect (ObjNum 1230 snap 0 seq_num 1230) 2014-07-01 17:12:49,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: expect (ObjNum 2193 snap 0 seq_num 2193) 2014-07-01 17:12:49,174.174 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:12:49,901.901 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:12:49,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3312: done (2 left) 2014-07-01 17:12:49,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3314: done (1 left) 2014-07-01 17:12:49,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3315: done (0 left) 2014-07-01 17:12:49,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: read oid 161 snap -1 2014-07-01 17:12:49,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3317: read oid 354 snap -1 2014-07-01 17:12:49,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: write oid 26 current snap is 381 2014-07-01 17:12:49,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: seq_num 1524 ranges {763659=474586,1855073=412391,2763658=1} 2014-07-01 17:12:49,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: writing mira0437250-26 from 763659 to 1238245 tid 1 2014-07-01 17:12:50,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: writing mira0437250-26 from 1855073 to 2267464 tid 2 2014-07-01 17:12:50,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: writing mira0437250-26 from 2763658 to 2763659 tid 3 2014-07-01 17:12:50,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: write oid 67 current snap is 381 2014-07-01 17:12:50,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: seq_num 1525 ranges {457865=446391,1566134=634010,2457864=1} 2014-07-01 17:12:50,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: writing mira0437250-67 from 457865 to 904256 tid 1 2014-07-01 17:12:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: writing mira0437250-67 from 1566134 to 2200144 tid 2 2014-07-01 17:12:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: writing mira0437250-67 from 2457864 to 2457865 tid 3 2014-07-01 17:12:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3320: snap_remove snap 347 2014-07-01 17:12:50,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3317: expect (ObjNum 1181 snap 250 seq_num 1181) 2014-07-01 17:12:50,423.423 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: expect (ObjNum 844 snap 141 seq_num 844) 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: finishing write tid 1 to mira0437250-26 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: finishing write tid 2 to mira0437250-26 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: finishing write tid 3 to mira0437250-26 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: finishing write tid 5 to mira0437250-26 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 1 to mira0437250-67 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 2 to mira0437250-67 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 3 to mira0437250-67 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 5 to mira0437250-67 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: finishing write tid 6 to mira0437250-67 2014-07-01 17:12:50,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 67 v 805 (ObjNum 1525 snap 381 seq_num 1525) dirty exists 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: finishing write tid 6 to mira0437250-26 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 1087 (ObjNum 1524 snap 381 seq_num 1524) dirty exists 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3316: done (4 left) 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3317: done (3 left) 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3318: done (2 left) 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3319: done (1 left) 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3320: done (0 left) 2014-07-01 17:12:50,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3321: snap_remove snap 272 2014-07-01 17:12:51,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3321: done (0 left) 2014-07-01 17:12:51,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3322: rollback oid 481 current snap is 381 2014-07-01 17:12:51,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 481 to 359 2014-07-01 17:12:51,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: write oid 284 current snap is 381 2014-07-01 17:12:51,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: seq_num 1526 ranges {554119=570643,1681290=472829} 2014-07-01 17:12:51,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0437250-284 from 554119 to 1124762 tid 1 2014-07-01 17:12:51,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: writing mira0437250-284 from 1681290 to 2154119 tid 2 2014-07-01 17:12:51,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: snap_remove snap 330 2014-07-01 17:12:51,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 1157 (ObjNum 693 snap 88 seq_num 693) dirty exists 2014-07-01 17:12:52,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3322: done (2 left) 2014-07-01 17:12:52,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3324: done (1 left) 2014-07-01 17:12:52,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: read oid 364 snap -1 2014-07-01 17:12:52,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: write oid 134 current snap is 381 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: seq_num 1527 ranges {498948=426095,1643997=774401,2898947=1} 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: writing mira0437250-134 from 498948 to 925043 tid 1 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: writing mira0437250-134 from 1643997 to 2418398 tid 2 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: writing mira0437250-134 from 2898947 to 2898948 tid 3 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3327: snap_create 2014-07-01 17:12:52,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 1 to mira0437250-134 2014-07-01 17:12:52,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 2 to mira0437250-134 2014-07-01 17:12:52,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 3 to mira0437250-134 2014-07-01 17:12:52,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 5 to mira0437250-134 2014-07-01 17:12:52,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: finishing write tid 6 to mira0437250-134 2014-07-01 17:12:52,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 134 v 1206 (ObjNum 1527 snap 381 seq_num 1527) dirty exists 2014-07-01 17:12:53,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3325: done (3 left) 2014-07-01 17:12:53,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3326: done (2 left) 2014-07-01 17:12:53,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3327: done (1 left) 2014-07-01 17:12:53,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: snap_remove snap 376 2014-07-01 17:12:53,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: done (3 left) 2014-07-01 17:12:53,324.324 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: done (2 left) 2014-07-01 17:12:53,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: done (1 left) 2014-07-01 17:12:53,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3768: done (0 left) 2014-07-01 17:12:53,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3769: read oid 121 snap -1 2014-07-01 17:12:53,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 17:12:53,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: seq_num 2307 ranges {6905856=700416,7606272=593920,8200192=524288,8724480=86016} 2014-07-01 17:12:53,398.398 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0435601-492 from 6905856 to 7606272 tid 1 2014-07-01 17:12:53,491.491 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0435601-492 from 7606272 to 8200192 tid 2 2014-07-01 17:12:53,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0435601-492 from 8200192 to 8724480 tid 3 2014-07-01 17:12:53,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0435601-492 from 8724480 to 8810496 tid 4 2014-07-01 17:12:53,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 226 current snap is 0 2014-07-01 17:12:53,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: seq_num 2308 ranges {7503872=573440,8077312=421888,8499200=753664,9252864=28672} 2014-07-01 17:12:53,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: writing mira0435601-226 from 7503872 to 8077312 tid 1 2014-07-01 17:12:53,504.504 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: writing mira0435601-226 from 8077312 to 8499200 tid 2 2014-07-01 17:12:53,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: writing mira0435601-226 from 8499200 to 9252864 tid 3 2014-07-01 17:12:53,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: writing mira0435601-226 from 9252864 to 9281536 tid 4 2014-07-01 17:12:53,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 301 current snap is 0 2014-07-01 17:12:53,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: seq_num 2309 ranges {3395584=647168,4042752=602112,4644864=200704} 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: writing mira0435601-301 from 3395584 to 4042752 tid 1 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: writing mira0435601-301 from 4042752 to 4644864 tid 2 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: writing mira0435601-301 from 4644864 to 4845568 tid 3 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 77 current snap is 0 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: seq_num 2310 ranges {3076096=520192,3596288=589824,4186112=212992} 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: writing mira0435601-77 from 3076096 to 3596288 tid 1 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: writing mira0435601-77 from 3596288 to 4186112 tid 2 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: writing mira0435601-77 from 4186112 to 4399104 tid 3 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3774: read oid 68 snap -1 2014-07-01 17:12:53,656.656 INFO:teuthology.task.rados.rados.0.mira043.stdout:3769: expect (ObjNum 893 snap 0 seq_num 893) 2014-07-01 17:12:53,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 1 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 2 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 3 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 4 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 6 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 7 to mira0435601-492 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 230 (ObjNum 2307 snap 0 seq_num 2307) dirty exists 2014-07-01 17:12:53,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3769: done (5 left) 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: done (4 left) 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3775: delete oid 367 current snap is 0 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: finishing write tid 1 to mira0435601-301 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: finishing write tid 2 to mira0435601-301 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: finishing write tid 3 to mira0435601-301 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: finishing write tid 5 to mira0435601-301 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: finishing write tid 6 to mira0435601-301 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 325 (ObjNum 2309 snap 0 seq_num 2309) dirty exists 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: finishing write tid 1 to mira0435601-77 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: finishing write tid 2 to mira0435601-77 2014-07-01 17:12:53,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: finishing write tid 3 to mira0435601-77 2014-07-01 17:12:53,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: finishing write tid 5 to mira0435601-77 2014-07-01 17:12:53,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: finishing write tid 6 to mira0435601-77 2014-07-01 17:12:53,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 238 (ObjNum 2310 snap 0 seq_num 2310) dirty exists 2014-07-01 17:12:53,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:3774: expect (ObjNum 1706 snap 0 seq_num 1706) 2014-07-01 17:12:54,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:3328: done (1 left) 2014-07-01 17:12:54,140.140 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: read oid 375 snap -1 2014-07-01 17:12:54,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: write oid 463 current snap is 382 2014-07-01 17:12:54,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: seq_num 1528 ranges {442754=549324,1757106=596020,3046156=555244,3642753=1} 2014-07-01 17:12:54,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: writing mira0437250-463 from 442754 to 992078 tid 1 2014-07-01 17:12:54,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: writing mira0437250-463 from 1757106 to 2353126 tid 2 2014-07-01 17:12:54,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: writing mira0437250-463 from 3046156 to 3601400 tid 3 2014-07-01 17:12:54,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: writing mira0437250-463 from 3642753 to 3642754 tid 4 2014-07-01 17:12:54,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: snap_create 2014-07-01 17:12:54,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 1 to mira0437250-463 2014-07-01 17:12:54,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 2 to mira0437250-463 2014-07-01 17:12:54,306.306 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: expect (ObjNum 1117 snap 224 seq_num 1117) 2014-07-01 17:12:54,368.368 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:12:55,106.106 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 14, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008213', u'skew': u'0.001456', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.004369', u'skew': u'0.001461', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:12:45.735760', u'name': u'c', u'avail_percent': 92, u'kb_total': 944797076, u'kb_avail': 878457832, u'health': u'HEALTH_OK', u'kb_used': 18323220, u'store_stats': {u'bytes_total': 5502633, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4978329}}, {u'last_updated': u'2014-07-01 08:11:56.013431', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883183652, u'health': u'HEALTH_OK', u'kb_used': 13597400, u'store_stats': {u'bytes_total': 10557509, u'bytes_log': 5177344, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5314613}}, {u'last_updated': u'2014-07-01 08:12:13.317437', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883119536, u'health': u'HEALTH_OK', u'kb_used': 13661516, u'store_stats': {u'bytes_total': 13703278, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5314654}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'16 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'26 pgs recovery_wait'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 426/2964 objects degraded (14.372%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 426, u'num_pgs': 534, u'data_bytes': 2500587385, u'degraded_total': 2964, u'bytes_used': 6318292992, u'version': 2965, u'pgs_by_state': [{u'count': 26, u'state_name': u'active+recovery_wait'}, {u'count': 492, u'state_name': u'active+clean'}, {u'count': 16, u'state_name': u'active+recovering'}], u'degraded_ratio': u'14.372', u'bytes_avail': 5991980752896}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1463, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:12:55,107.107 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:12:55,829.829 INFO:teuthology.task.thrashosds.ceph_manager:no progress seen, keeping timeout for now 2014-07-01 17:12:55,829.829 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 3 to mira0437250-463 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 4 to mira0437250-463 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 6 to mira0437250-463 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: finishing write tid 7 to mira0437250-463 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 1417 (ObjNum 1528 snap 382 seq_num 1528) dirty exists 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 1 to mira0437250-284 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 2 to mira0437250-284 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 4 to mira0437250-284 2014-07-01 17:12:55,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: finishing write tid 5 to mira0437250-284 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 1601 (ObjNum 1526 snap 381 seq_num 1526) dirty exists 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3323: done (3 left) 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3329: done (2 left) 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3330: done (1 left) 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3331: done (0 left) 2014-07-01 17:12:55,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: write oid 187 current snap is 383 2014-07-01 17:12:55,919.919 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: seq_num 1529 ranges {655360=770728,2202118=514791,3178261=677099} 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: writing mira0437250-187 from 655360 to 1426088 tid 1 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: writing mira0437250-187 from 2202118 to 2716909 tid 2 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: writing mira0437250-187 from 3178261 to 3855360 tid 3 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: read oid 303 snap -1 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3334: snap_remove snap 372 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: expect (ObjNum 1495 snap 368 seq_num 1495) 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: finishing write tid 1 to mira0437250-187 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: finishing write tid 2 to mira0437250-187 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: finishing write tid 3 to mira0437250-187 2014-07-01 17:12:56,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: finishing write tid 5 to mira0437250-187 2014-07-01 17:12:56,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: finishing write tid 6 to mira0437250-187 2014-07-01 17:12:56,112.112 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 187 v 1497 (ObjNum 1529 snap 383 seq_num 1529) dirty exists 2014-07-01 17:12:56,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3332: done (2 left) 2014-07-01 17:12:56,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3333: done (1 left) 2014-07-01 17:12:56,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3334: done (0 left) 2014-07-01 17:12:56,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3335: snap_create 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 1 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 2 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 3 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 4 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 6 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: finishing write tid 7 to mira0435601-226 2014-07-01 17:12:56,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 148 (ObjNum 2308 snap 0 seq_num 2308) dirty exists 2014-07-01 17:12:57,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:3335: done (0 left) 2014-07-01 17:12:57,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3336: delete oid 164 current snap is 384 2014-07-01 17:12:57,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:3336: done (0 left) 2014-07-01 17:12:57,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: write oid 492 current snap is 384 2014-07-01 17:12:57,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: seq_num 1531 ranges {543995=749409,1343994=1} 2014-07-01 17:12:57,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: writing mira0437250-492 from 543995 to 1293404 tid 1 2014-07-01 17:12:57,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: writing mira0437250-492 from 1343994 to 1343995 tid 2 2014-07-01 17:12:57,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:3338: rollback oid 452 current snap is 384 2014-07-01 17:12:57,226.226 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 452 to 329 2014-07-01 17:12:57,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3339: delete oid 400 current snap is 384 2014-07-01 17:12:58,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3339: done (2 left) 2014-07-01 17:12:58,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3340: read oid 313 snap -1 2014-07-01 17:12:58,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: read oid 369 snap -1 2014-07-01 17:12:58,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:3342: rollback oid 222 current snap is 384 2014-07-01 17:12:58,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 222 to 359 2014-07-01 17:12:58,137.137 INFO:teuthology.task.rados.rados.0.mira043.stdout:3343: snap_create 2014-07-01 17:12:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 1 to mira0437250-492 2014-07-01 17:12:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 2 to mira0437250-492 2014-07-01 17:12:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 4 to mira0437250-492 2014-07-01 17:12:58,138.138 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 222 v 0 (ObjNum 1 snap 1 seq_num 16088064) dirty dne 2014-07-01 17:12:58,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 1305 (ObjNum 800 snap 129 seq_num 800) dirty exists 2014-07-01 17:12:58,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: finishing write tid 5 to mira0437250-492 2014-07-01 17:12:58,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 1302 (ObjNum 1531 snap 384 seq_num 1531) dirty exists 2014-07-01 17:12:58,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: expect (ObjNum 1209 snap 254 seq_num 1209) 2014-07-01 17:12:58,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: done (4 left) 2014-07-01 17:12:58,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: done (3 left) 2014-07-01 17:12:58,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: done (2 left) 2014-07-01 17:12:58,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3774: done (1 left) 2014-07-01 17:12:58,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3775: done (0 left) 2014-07-01 17:12:58,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 314 current snap is 0 2014-07-01 17:12:58,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: seq_num 2312 ranges {724992=16384} 2014-07-01 17:12:58,395.395 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: writing mira0435601-314 from 724992 to 741376 tid 1 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: read oid 107 snap -1 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 152 current snap is 0 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: seq_num 2313 ranges {2998272=741376} 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: writing mira0435601-152 from 2998272 to 3739648 tid 1 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3779: read oid 241 snap -1 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: finishing write tid 1 to mira0435601-314 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: finishing write tid 3 to mira0435601-314 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: finishing write tid 4 to mira0435601-314 2014-07-01 17:12:58,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 242 (ObjNum 2312 snap 0 seq_num 2312) dirty exists 2014-07-01 17:12:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: done (3 left) 2014-07-01 17:12:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: delete oid 485 current snap is 0 2014-07-01 17:12:58,467.467 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: expect (ObjNum 1632 snap 0 seq_num 1632) 2014-07-01 17:13:00,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3340: expect (ObjNum 1087 snap 218 seq_num 1087) 2014-07-01 17:13:01,136.136 INFO:teuthology.task.rados.rados.0.mira043.stdout:3779: expect (ObjNum 1657 snap 0 seq_num 1657) 2014-07-01 17:13:03,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3337: done (5 left) 2014-07-01 17:13:03,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3338: done (4 left) 2014-07-01 17:13:03,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3340: done (3 left) 2014-07-01 17:13:03,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3341: done (2 left) 2014-07-01 17:13:03,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3342: done (1 left) 2014-07-01 17:13:03,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3343: done (0 left) 2014-07-01 17:13:03,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: write oid 216 current snap is 385 2014-07-01 17:13:03,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: seq_num 1533 ranges {700407=445465,1670102=581044,2300406=1} 2014-07-01 17:13:03,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: writing mira0437250-216 from 700407 to 1145872 tid 1 2014-07-01 17:13:03,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: writing mira0437250-216 from 1670102 to 2251146 tid 2 2014-07-01 17:13:03,362.362 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: writing mira0437250-216 from 2300406 to 2300407 tid 3 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: snap_create 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: finishing write tid 1 to mira0437250-216 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: finishing write tid 2 to mira0437250-216 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: finishing write tid 3 to mira0437250-216 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: finishing write tid 5 to mira0437250-216 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: finishing write tid 6 to mira0437250-216 2014-07-01 17:13:03,363.363 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 216 v 1754 (ObjNum 1533 snap 385 seq_num 1533) dirty exists 2014-07-01 17:13:04,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3344: done (1 left) 2014-07-01 17:13:04,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3345: done (0 left) 2014-07-01 17:13:04,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: read oid 296 snap -1 2014-07-01 17:13:04,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3347: read oid 74 snap 377 2014-07-01 17:13:04,196.196 INFO:teuthology.task.rados.rados.0.mira043.stdout:3348: snap_create 2014-07-01 17:13:04,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3347: expect (ObjNum 1280 snap 284 seq_num 1280) 2014-07-01 17:13:04,939.939 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:05,355.355 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: expect (ObjNum 1500 snap 369 seq_num 1500) 2014-07-01 17:13:05,755.755 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:05,970.970 INFO:teuthology.task.rados.rados.0.mira043.stdout:3346: done (2 left) 2014-07-01 17:13:05,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3347: done (1 left) 2014-07-01 17:13:05,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3348: done (0 left) 2014-07-01 17:13:05,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: delete oid 396 current snap is 387 2014-07-01 17:13:07,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3349: done (0 left) 2014-07-01 17:13:07,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3350: rollback oid 298 current snap is 387 2014-07-01 17:13:07,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 298 to 385 2014-07-01 17:13:07,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3351: snap_create 2014-07-01 17:13:07,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 0 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:13:08,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3350: done (1 left) 2014-07-01 17:13:08,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3351: done (0 left) 2014-07-01 17:13:08,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: write oid 377 current snap is 388 2014-07-01 17:13:08,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: seq_num 1535 ranges {739442=424146,1744894=472858,2648013=91429} 2014-07-01 17:13:08,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: writing mira0437250-377 from 739442 to 1163588 tid 1 2014-07-01 17:13:08,258.258 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: writing mira0437250-377 from 1744894 to 2217752 tid 2 2014-07-01 17:13:08,263.263 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: writing mira0437250-377 from 2648013 to 2739442 tid 3 2014-07-01 17:13:08,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: write oid 82 current snap is 388 2014-07-01 17:13:08,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: seq_num 1536 ranges {684272=548398,1860050=543053,2850262=485281,3484271=1} 2014-07-01 17:13:08,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: writing mira0437250-82 from 684272 to 1232670 tid 1 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: writing mira0437250-82 from 1860050 to 2403103 tid 2 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: writing mira0437250-82 from 2850262 to 3335543 tid 3 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: writing mira0437250-82 from 3484271 to 3484272 tid 4 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3354: delete oid 177 current snap is 388 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: finishing write tid 1 to mira0437250-377 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: finishing write tid 2 to mira0437250-377 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: finishing write tid 3 to mira0437250-377 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: finishing write tid 5 to mira0437250-377 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: finishing write tid 6 to mira0437250-377 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 1200 (ObjNum 1535 snap 388 seq_num 1535) dirty exists 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 1 to mira0437250-82 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 2 to mira0437250-82 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3352: done (2 left) 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3354: done (1 left) 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: read oid 144 snap 344 2014-07-01 17:13:08,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: delete oid 430 current snap is 388 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 3 to mira0437250-82 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 4 to mira0437250-82 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3355: done (2 left) 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3356: done (1 left) 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: read oid 73 snap -1 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: snap_create 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 6 to mira0437250-82 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: finishing write tid 7 to mira0437250-82 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 82 v 1481 (ObjNum 1536 snap 388 seq_num 1536) dirty exists 2014-07-01 17:13:08,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: expect (ObjNum 1514 snap 376 seq_num 1514) 2014-07-01 17:13:08,667.667 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 1 to mira0435601-152 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 3 to mira0435601-152 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 4 to mira0435601-152 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 180 (ObjNum 2313 snap 0 seq_num 2313) dirty exists 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: done (3 left) 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: done (2 left) 2014-07-01 17:13:08,668.668 INFO:teuthology.task.rados.rados.0.mira043.stdout:3779: done (1 left) 2014-07-01 17:13:08,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: done (0 left) 2014-07-01 17:13:08,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 285 current snap is 0 2014-07-01 17:13:08,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: seq_num 2315 ranges {0=638976,638976=798720,1437696=405504} 2014-07-01 17:13:08,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: writing mira0435601-285 from 0 to 638976 tid 1 2014-07-01 17:13:08,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: writing mira0435601-285 from 638976 to 1437696 tid 2 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: writing mira0435601-285 from 1437696 to 1843200 tid 3 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3782: read oid 360 snap -1 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: finishing write tid 1 to mira0435601-285 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 228 current snap is 0 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: seq_num 2316 ranges {4161536=512000,4673536=786432,5459968=417792} 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: writing mira0435601-228 from 4161536 to 4673536 tid 1 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: writing mira0435601-228 from 4673536 to 5459968 tid 2 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: writing mira0435601-228 from 5459968 to 5877760 tid 3 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3784: delete oid 1 current snap is 0 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: finishing write tid 2 to mira0435601-285 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: finishing write tid 3 to mira0435601-285 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: finishing write tid 5 to mira0435601-285 2014-07-01 17:13:08,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: finishing write tid 6 to mira0435601-285 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 213 (ObjNum 2315 snap 0 seq_num 2315) dirty exists 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 1 to mira0435601-228 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 2 to mira0435601-228 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: done (3 left) 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3782: done (2 left) 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3784: done (1 left) 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3785: read oid 273 snap -1 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3786: read oid 371 snap -1 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 267 current snap is 0 2014-07-01 17:13:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: seq_num 2318 ranges {5414912=458752,5873664=548864,6422528=253952} 2014-07-01 17:13:08,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: writing mira0435601-267 from 5414912 to 5873664 tid 1 2014-07-01 17:13:08,875.875 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: writing mira0435601-267 from 5873664 to 6422528 tid 2 2014-07-01 17:13:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: writing mira0435601-267 from 6422528 to 6676480 tid 3 2014-07-01 17:13:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 127 current snap is 0 2014-07-01 17:13:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: seq_num 2319 ranges {4767744=733184,5500928=401408} 2014-07-01 17:13:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: writing mira0435601-127 from 4767744 to 5500928 tid 1 2014-07-01 17:13:09,030.030 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: writing mira0435601-127 from 5500928 to 5902336 tid 2 2014-07-01 17:13:09,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 348 current snap is 0 2014-07-01 17:13:09,031.031 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: seq_num 2320 ranges {4825088=757760,5582848=610304,6193152=536576,6729728=458752} 2014-07-01 17:13:09,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: writing mira0435601-348 from 4825088 to 5582848 tid 1 2014-07-01 17:13:09,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: writing mira0435601-348 from 5582848 to 6193152 tid 2 2014-07-01 17:13:09,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: writing mira0435601-348 from 6193152 to 6729728 tid 3 2014-07-01 17:13:09,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: writing mira0435601-348 from 6729728 to 7188480 tid 4 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3790: delete oid 390 current snap is 0 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 3 to mira0435601-228 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 5 to mira0435601-228 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 6 to mira0435601-228 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 228 v 178 (ObjNum 2316 snap 0 seq_num 2316) dirty exists 2014-07-01 17:13:09,235.235 INFO:teuthology.task.rados.rados.0.mira043.stdout:3786: expect (ObjNum 1848 snap 0 seq_num 1848) 2014-07-01 17:13:09,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3353: done (2 left) 2014-07-01 17:13:09,247.247 INFO:teuthology.task.rados.rados.0.mira043.stdout:3357: done (1 left) 2014-07-01 17:13:09,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3358: done (0 left) 2014-07-01 17:13:09,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: snap_remove snap 374 2014-07-01 17:13:10,066.066 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:13:10,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3785: expect (ObjNum 2093 snap 0 seq_num 2093) 2014-07-01 17:13:10,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:3359: done (0 left) 2014-07-01 17:13:10,239.239 INFO:teuthology.task.rados.rados.0.mira043.stdout:3360: snap_remove snap 303 2014-07-01 17:13:10,769.769 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 14, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008213', u'skew': u'0.001456', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.004369', u'skew': u'0.001461', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:12:45.735760', u'name': u'c', u'avail_percent': 92, u'kb_total': 944797076, u'kb_avail': 878457832, u'health': u'HEALTH_OK', u'kb_used': 18323220, u'store_stats': {u'bytes_total': 5502633, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4978329}}, {u'last_updated': u'2014-07-01 08:12:56.013682', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882997112, u'health': u'HEALTH_OK', u'kb_used': 13783940, u'store_stats': {u'bytes_total': 8122905, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4977161}}, {u'last_updated': u'2014-07-01 08:12:13.317437', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 883119536, u'health': u'HEALTH_OK', u'kb_used': 13661516, u'store_stats': {u'bytes_total': 13703278, u'bytes_log': 8323072, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5314654}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'5 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 63/2840 objects degraded (2.218%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'degraded_objects': 63, u'num_pgs': 534, u'data_bytes': 2434030361, u'degraded_total': 2840, u'bytes_used': 6099410944, u'version': 2981, u'pgs_by_state': [{u'count': 529, u'state_name': u'active+clean'}, {u'count': 5, u'state_name': u'active+recovering'}], u'degraded_ratio': u'2.218', u'bytes_avail': 5992199634944}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1472, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:13:10,769.769 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:13:11,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3360: done (0 left) 2014-07-01 17:13:11,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3361: delete oid 169 current snap is 389 2014-07-01 17:13:11,370.370 INFO:teuthology.task.rados.rados.0.mira043.stdout:3361: done (0 left) 2014-07-01 17:13:11,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: snap_create 2014-07-01 17:13:11,520.520 INFO:teuthology.task.thrashosds.ceph_manager:making progress, resetting timeout 2014-07-01 17:13:11,520.520 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:12,271.271 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:12,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3362: done (0 left) 2014-07-01 17:13:12,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: write oid 166 current snap is 390 2014-07-01 17:13:12,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: seq_num 1540 ranges {480743=400000} 2014-07-01 17:13:12,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: writing mira0437250-166 from 480743 to 880743 tid 1 2014-07-01 17:13:12,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: read oid 330 snap 385 2014-07-01 17:13:12,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: write oid 139 current snap is 390 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: seq_num 1541 ranges {425573=595964,1625572=1} 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0437250-139 from 425573 to 1021537 tid 1 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: writing mira0437250-139 from 1625572 to 1625573 tid 2 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: write oid 372 current snap is 390 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: seq_num 1542 ranges {485730=567769,1851084=502145,2885729=1} 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0437250-372 from 485730 to 1053499 tid 1 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0437250-372 from 1851084 to 2353229 tid 2 2014-07-01 17:13:12,522.522 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: writing mira0437250-372 from 2885729 to 2885730 tid 3 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: write oid 339 current snap is 390 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: seq_num 1543 ranges {580960=574645,1599264=420916,2580959=1} 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: writing mira0437250-339 from 580960 to 1155605 tid 1 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: writing mira0437250-339 from 1599264 to 2020180 tid 2 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: writing mira0437250-339 from 2580959 to 2580960 tid 3 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: read oid 480 snap 361 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: finishing write tid 1 to mira0437250-166 2014-07-01 17:13:12,523.523 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: expect (ObjNum 1375 snap 312 seq_num 1375) 2014-07-01 17:13:12,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: finishing write tid 1 to mira0435601-267 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: finishing write tid 2 to mira0435601-267 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: finishing write tid 3 to mira0435601-267 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: finishing write tid 5 to mira0435601-267 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: finishing write tid 6 to mira0435601-267 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 267 v 359 (ObjNum 2318 snap 0 seq_num 2318) dirty exists 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: finishing write tid 1 to mira0435601-127 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: done (6 left) 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3785: done (5 left) 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3786: done (4 left) 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: done (3 left) 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3790: done (2 left) 2014-07-01 17:13:12,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 309 current snap is 0 2014-07-01 17:13:12,812.812 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: seq_num 2322 ranges {2727936=552960} 2014-07-01 17:13:12,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: writing mira0435601-309 from 2727936 to 3280896 tid 1 2014-07-01 17:13:12,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 129 current snap is 0 2014-07-01 17:13:12,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: seq_num 2323 ranges {5464064=577536,6041600=606208,6647808=598016} 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0435601-129 from 5464064 to 6041600 tid 1 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0435601-129 from 6041600 to 6647808 tid 2 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0435601-129 from 6647808 to 7245824 tid 3 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 215 current snap is 0 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: seq_num 2324 ranges {7430144=294912} 2014-07-01 17:13:13,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: finishing write tid 3 to mira0437250-166 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: finishing write tid 4 to mira0437250-166 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 717 (ObjNum 1540 snap 390 seq_num 1540) dirty exists 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 1 to mira0437250-139 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 2 to mira0437250-139 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 4 to mira0437250-139 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: finishing write tid 5 to mira0437250-139 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 139 v 1023 (ObjNum 1541 snap 390 seq_num 1541) dirty exists 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 1 to mira0437250-372 2014-07-01 17:13:13,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3363: done (5 left) 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3364: done (4 left) 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3365: done (3 left) 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3369: snap_create 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 2 to mira0437250-372 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 3 to mira0437250-372 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 5 to mira0437250-372 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: finishing write tid 6 to mira0437250-372 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 372 v 872 (ObjNum 1542 snap 390 seq_num 1542) dirty exists 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: finishing write tid 1 to mira0437250-339 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: finishing write tid 2 to mira0437250-339 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: finishing write tid 3 to mira0437250-339 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: finishing write tid 5 to mira0437250-339 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: expect (ObjNum 1231 snap 269 seq_num 1231) 2014-07-01 17:13:13,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: writing mira0435601-215 from 7430144 to 7725056 tid 1 2014-07-01 17:13:13,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 221 current snap is 0 2014-07-01 17:13:13,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: seq_num 2325 ranges {2781184=724992,3506176=770048,4276224=32768} 2014-07-01 17:13:13,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0435601-221 from 2781184 to 3506176 tid 1 2014-07-01 17:13:13,184.184 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0435601-221 from 3506176 to 4276224 tid 2 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0435601-221 from 4276224 to 4308992 tid 3 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 110 current snap is 0 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: seq_num 2326 ranges {3780608=348160} 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: writing mira0435601-110 from 3780608 to 4128768 tid 1 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3796: read oid 442 snap -1 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: finishing write tid 2 to mira0435601-127 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: finishing write tid 4 to mira0435601-127 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: finishing write tid 5 to mira0435601-127 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 127 v 215 (ObjNum 2319 snap 0 seq_num 2319) dirty exists 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 1 to mira0435601-348 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: done (7 left) 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: read oid 103 snap -1 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 2 to mira0435601-348 2014-07-01 17:13:13,185.185 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 3 to mira0435601-348 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 4 to mira0435601-348 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 6 to mira0435601-348 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: finishing write tid 7 to mira0435601-348 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 350 (ObjNum 2320 snap 0 seq_num 2320) dirty exists 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 1 to mira0435601-309 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 3 to mira0435601-309 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 4 to mira0435601-309 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 299 (ObjNum 2322 snap 0 seq_num 2322) dirty exists 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 1 to mira0435601-129 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 2 to mira0435601-129 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 3 to mira0435601-129 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 5 to mira0435601-129 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 6 to mira0435601-129 2014-07-01 17:13:13,187.187 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 129 v 320 (ObjNum 2323 snap 0 seq_num 2323) dirty exists 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: finishing write tid 1 to mira0435601-215 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: done (7 left) 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: done (6 left) 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: done (5 left) 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 188 current snap is 0 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: seq_num 2327 ranges {4026368=516096,4542464=585728,5128192=606208,5734400=413696} 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: writing mira0435601-188 from 4026368 to 4542464 tid 1 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: writing mira0435601-188 from 4542464 to 5128192 tid 2 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: writing mira0435601-188 from 5128192 to 5734400 tid 3 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: writing mira0435601-188 from 5734400 to 6148096 tid 4 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 220 current snap is 0 2014-07-01 17:13:13,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: seq_num 2328 ranges {3784704=544768,4329472=401408} 2014-07-01 17:13:13,217.217 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: writing mira0435601-220 from 3784704 to 4329472 tid 1 2014-07-01 17:13:13,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: writing mira0435601-220 from 4329472 to 4730880 tid 2 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 318 current snap is 0 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: seq_num 2329 ranges {0=569344,569344=598016,1167360=659456,1826816=348160} 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: writing mira0435601-318 from 0 to 569344 tid 1 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: writing mira0435601-318 from 569344 to 1167360 tid 2 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: writing mira0435601-318 from 1167360 to 1826816 tid 3 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: writing mira0435601-318 from 1826816 to 2174976 tid 4 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 299 current snap is 0 2014-07-01 17:13:13,352.352 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: seq_num 2330 ranges {3461120=442368,3903488=581632,4485120=532480,5017600=491520} 2014-07-01 17:13:13,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: writing mira0435601-299 from 3461120 to 3903488 tid 1 2014-07-01 17:13:13,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: writing mira0435601-299 from 3903488 to 4485120 tid 2 2014-07-01 17:13:13,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: writing mira0435601-299 from 4485120 to 5017600 tid 3 2014-07-01 17:13:13,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: writing mira0435601-299 from 5017600 to 5509120 tid 4 2014-07-01 17:13:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 241 current snap is 0 2014-07-01 17:13:13,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: seq_num 2331 ranges {8634368=364544} 2014-07-01 17:13:13,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: writing mira0435601-241 from 8634368 to 8998912 tid 1 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 33 current snap is 0 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: seq_num 2332 ranges {2138112=790528,2928640=798720,3727360=4096} 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: writing mira0435601-33 from 2138112 to 2928640 tid 1 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: writing mira0435601-33 from 2928640 to 3727360 tid 2 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: writing mira0435601-33 from 3727360 to 3731456 tid 3 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3804: delete oid 44 current snap is 0 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: finishing write tid 3 to mira0435601-215 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: finishing write tid 4 to mira0435601-215 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 215 v 355 (ObjNum 2324 snap 0 seq_num 2324) dirty exists 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 1 to mira0435601-221 2014-07-01 17:13:13,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 2 to mira0435601-221 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 3 to mira0435601-221 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 5 to mira0435601-221 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 6 to mira0435601-221 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 221 v 377 (ObjNum 2325 snap 0 seq_num 2325) dirty exists 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 1 to mira0435601-110 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 3 to mira0435601-110 2014-07-01 17:13:13,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: expect (ObjNum 1643 snap 0 seq_num 1643) 2014-07-01 17:13:14,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: finishing write tid 6 to mira0437250-339 2014-07-01 17:13:14,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 339 v 1398 (ObjNum 1543 snap 390 seq_num 1543) dirty exists 2014-07-01 17:13:14,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3366: done (3 left) 2014-07-01 17:13:14,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3367: done (2 left) 2014-07-01 17:13:14,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3368: done (1 left) 2014-07-01 17:13:14,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3369: done (0 left) 2014-07-01 17:13:14,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3370: read oid 379 snap 389 2014-07-01 17:13:14,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3371: snap_remove snap 315 2014-07-01 17:13:14,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3370: done (1 left) 2014-07-01 17:13:14,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3371: done (0 left) 2014-07-01 17:13:14,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3372: snap_create 2014-07-01 17:13:14,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 4 to mira0435601-110 2014-07-01 17:13:14,716.716 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 110 v 215 (ObjNum 2326 snap 0 seq_num 2326) dirty exists 2014-07-01 17:13:14,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 1 to mira0435601-188 2014-07-01 17:13:14,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 2 to mira0435601-188 2014-07-01 17:13:14,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3796: expect (ObjNum 1763 snap 0 seq_num 1763) 2014-07-01 17:13:15,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3372: done (0 left) 2014-07-01 17:13:15,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: write oid 182 current snap is 392 2014-07-01 17:13:15,357.357 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: seq_num 1544 ranges {525790=547473,1527924=642535,2903970=421820} 2014-07-01 17:13:15,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: writing mira0437250-182 from 525790 to 1073263 tid 1 2014-07-01 17:13:15,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: writing mira0437250-182 from 1527924 to 2170459 tid 2 2014-07-01 17:13:15,414.414 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: writing mira0437250-182 from 2903970 to 3325790 tid 3 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: write oid 213 current snap is 392 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: seq_num 1545 ranges {69595=1} 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: writing mira0437250-213 from 69595 to 69596 tid 1 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: read oid 142 snap 389 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: finishing write tid 1 to mira0437250-182 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: finishing write tid 2 to mira0437250-182 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: write oid 84 current snap is 392 2014-07-01 17:13:15,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: seq_num 1546 ranges {565849=642506,1755164=534349,2940468=568674,3765848=1} 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: writing mira0437250-84 from 565849 to 1208355 tid 1 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: writing mira0437250-84 from 1755164 to 2289513 tid 2 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: writing mira0437250-84 from 2940468 to 3509142 tid 3 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: writing mira0437250-84 from 3765848 to 3765849 tid 4 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3377: read oid 412 snap -1 2014-07-01 17:13:15,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: expect (ObjNum 141 snap 0 seq_num 141) 2014-07-01 17:13:15,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: finishing write tid 3 to mira0437250-182 2014-07-01 17:13:15,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: finishing write tid 5 to mira0437250-182 2014-07-01 17:13:15,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: finishing write tid 6 to mira0437250-182 2014-07-01 17:13:15,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 1536 (ObjNum 1544 snap 392 seq_num 1544) dirty exists 2014-07-01 17:13:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 1 to mira0437250-213 2014-07-01 17:13:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 3 to mira0437250-213 2014-07-01 17:13:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: finishing write tid 4 to mira0437250-213 2014-07-01 17:13:15,799.799 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 213 v 1539 (ObjNum 1545 snap 392 seq_num 1545) dirty exists 2014-07-01 17:13:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 1 to mira0437250-84 2014-07-01 17:13:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 2 to mira0437250-84 2014-07-01 17:13:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 3 to mira0437250-84 2014-07-01 17:13:15,800.800 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 4 to mira0437250-84 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 6 to mira0437250-84 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: finishing write tid 7 to mira0437250-84 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 84 v 1249 (ObjNum 1546 snap 392 seq_num 1546) dirty exists 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3373: done (4 left) 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3374: done (3 left) 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3375: done (2 left) 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3376: done (1 left) 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3377: done (0 left) 2014-07-01 17:13:15,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: write oid 25 current snap is 392 2014-07-01 17:13:15,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: seq_num 1547 ranges {627031=400000} 2014-07-01 17:13:15,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: writing mira0437250-25 from 627031 to 1027031 tid 1 2014-07-01 17:13:15,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3379: snap_create 2014-07-01 17:13:15,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 1 to mira0437250-25 2014-07-01 17:13:15,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 3 to mira0437250-25 2014-07-01 17:13:15,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: finishing write tid 4 to mira0437250-25 2014-07-01 17:13:15,965.965 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 25 v 1057 (ObjNum 1547 snap 392 seq_num 1547) dirty exists 2014-07-01 17:13:16,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3378: done (1 left) 2014-07-01 17:13:16,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3379: done (0 left) 2014-07-01 17:13:16,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: read oid 493 snap -1 2014-07-01 17:13:16,367.367 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: read oid 83 snap -1 2014-07-01 17:13:16,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: read oid 265 snap -1 2014-07-01 17:13:16,368.368 INFO:teuthology.task.rados.rados.0.mira043.stdout:3383: snap_create 2014-07-01 17:13:16,532.532 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: expect (ObjNum 1457 snap 353 seq_num 1457) 2014-07-01 17:13:16,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: expect (ObjNum 1077 snap 216 seq_num 1077) 2014-07-01 17:13:17,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3380: done (3 left) 2014-07-01 17:13:17,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3381: done (2 left) 2014-07-01 17:13:17,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3382: done (1 left) 2014-07-01 17:13:17,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3383: done (0 left) 2014-07-01 17:13:17,595.595 INFO:teuthology.task.rados.rados.0.mira043.stdout:3384: snap_create 2014-07-01 17:13:18,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3384: done (0 left) 2014-07-01 17:13:18,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: rollback oid 316 current snap is 395 2014-07-01 17:13:18,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 316 to 325 2014-07-01 17:13:18,379.379 INFO:teuthology.task.rados.rados.0.mira043.stdout:3386: delete oid 399 current snap is 395 2014-07-01 17:13:18,386.386 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 316 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:13:18,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 3 to mira0435601-188 2014-07-01 17:13:18,387.387 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 4 to mira0435601-188 2014-07-01 17:13:18,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 6 to mira0435601-188 2014-07-01 17:13:18,388.388 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: finishing write tid 7 to mira0435601-188 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 188 v 165 (ObjNum 2327 snap 0 seq_num 2327) dirty exists 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: finishing write tid 1 to mira0435601-220 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: finishing write tid 2 to mira0435601-220 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: finishing write tid 4 to mira0435601-220 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: finishing write tid 5 to mira0435601-220 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 220 v 213 (ObjNum 2328 snap 0 seq_num 2328) dirty exists 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 1 to mira0435601-318 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 2 to mira0435601-318 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 3 to mira0435601-318 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 4 to mira0435601-318 2014-07-01 17:13:18,545.545 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 6 to mira0435601-318 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: finishing write tid 7 to mira0435601-318 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 318 v 186 (ObjNum 2329 snap 0 seq_num 2329) dirty exists 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 1 to mira0435601-299 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 2 to mira0435601-299 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 3 to mira0435601-299 2014-07-01 17:13:18,546.546 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 4 to mira0435601-299 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 6 to mira0435601-299 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: finishing write tid 7 to mira0435601-299 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 325 (ObjNum 2330 snap 0 seq_num 2330) dirty exists 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: finishing write tid 1 to mira0435601-241 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: finishing write tid 3 to mira0435601-241 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: finishing write tid 4 to mira0435601-241 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 241 v 266 (ObjNum 2331 snap 0 seq_num 2331) dirty exists 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: finishing write tid 1 to mira0435601-33 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: finishing write tid 2 to mira0435601-33 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: finishing write tid 3 to mira0435601-33 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: finishing write tid 5 to mira0435601-33 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: finishing write tid 6 to mira0435601-33 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 363 (ObjNum 2332 snap 0 seq_num 2332) dirty exists 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: done (11 left) 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: done (10 left) 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: done (9 left) 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3796: done (8 left) 2014-07-01 17:13:18,547.547 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: done (7 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: done (6 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: done (5 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: done (4 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: done (3 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: done (2 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: done (1 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3804: done (0 left) 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: seq_num 2334 ranges {0=532480,532480=561152,1093632=241664} 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: writing mira0435601-288 from 0 to 532480 tid 1 2014-07-01 17:13:18,552.552 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: writing mira0435601-288 from 532480 to 1093632 tid 2 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: writing mira0435601-288 from 1093632 to 1335296 tid 3 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3806: delete oid 374 current snap is 0 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: finishing write tid 1 to mira0435601-288 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3806: done (1 left) 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 211 current snap is 0 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: seq_num 2336 ranges {1961984=483328,2445312=401408} 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3385: done (1 left) 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3386: done (0 left) 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: write oid 74 current snap is 395 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: seq_num 1549 ranges {667090=745439,1467089=1} 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: writing mira0437250-74 from 667090 to 1412529 tid 1 2014-07-01 17:13:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: writing mira0437250-74 from 1467089 to 1467090 tid 2 2014-07-01 17:13:18,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:3388: rollback oid 152 current snap is 395 2014-07-01 17:13:18,554.554 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 152 to 380 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3389: snap_create 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 1435 (ObjNum 1413 snap 328 seq_num 1413) dirty exists 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: finishing write tid 1 to mira0437250-74 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: finishing write tid 2 to mira0437250-74 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: finishing write tid 4 to mira0437250-74 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: finishing write tid 5 to mira0437250-74 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 2140 (ObjNum 1549 snap 395 seq_num 1549) dirty exists 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: writing mira0435601-211 from 1961984 to 2445312 tid 1 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: writing mira0435601-211 from 2445312 to 2846720 tid 2 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 484 current snap is 0 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: seq_num 2337 ranges {3358720=753664} 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: writing mira0435601-484 from 3358720 to 4112384 tid 1 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3809: read oid 191 snap -1 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3810: read oid 446 snap -1 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: finishing write tid 2 to mira0435601-288 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: finishing write tid 3 to mira0435601-288 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: finishing write tid 5 to mira0435601-288 2014-07-01 17:13:18,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: finishing write tid 6 to mira0435601-288 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 386 (ObjNum 2334 snap 0 seq_num 2334) dirty exists 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: finishing write tid 1 to mira0435601-211 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: finishing write tid 2 to mira0435601-211 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: finishing write tid 4 to mira0435601-211 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: finishing write tid 5 to mira0435601-211 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 211 v 167 (ObjNum 2336 snap 0 seq_num 2336) dirty exists 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: done (4 left) 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: done (3 left) 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 418 current snap is 0 2014-07-01 17:13:18,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: seq_num 2338 ranges {3485696=782336,4268032=544768,4812800=425984,5238784=233472} 2014-07-01 17:13:18,649.649 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: writing mira0435601-418 from 3485696 to 4268032 tid 1 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: writing mira0435601-418 from 4268032 to 4812800 tid 2 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: writing mira0435601-418 from 4812800 to 5238784 tid 3 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: writing mira0435601-418 from 5238784 to 5472256 tid 4 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3812: read oid 332 snap -1 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3810: expect (ObjNum 2111 snap 0 seq_num 2111) 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: finishing write tid 1 to mira0435601-484 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: finishing write tid 3 to mira0435601-484 2014-07-01 17:13:18,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 1 to mira0435601-418 2014-07-01 17:13:18,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3809: expect (ObjNum 1153 snap 0 seq_num 1153) 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3387: done (2 left) 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3388: done (1 left) 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3389: done (0 left) 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: read oid 207 snap -1 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: write oid 250 current snap is 396 2014-07-01 17:13:19,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: seq_num 1550 ranges {609872=717243,1987052=222820} 2014-07-01 17:13:19,409.409 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:19,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: writing mira0437250-250 from 609872 to 1327115 tid 1 2014-07-01 17:13:19,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: writing mira0437250-250 from 1987052 to 2209872 tid 2 2014-07-01 17:13:19,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3392: rollback oid 296 current snap is 396 2014-07-01 17:13:19,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 296 to 373 2014-07-01 17:13:19,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3390: done (2 left) 2014-07-01 17:13:19,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3393: snap_create 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: finishing write tid 1 to mira0437250-250 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: finishing write tid 2 to mira0437250-250 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: finishing write tid 4 to mira0437250-250 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: finishing write tid 5 to mira0437250-250 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 916 (ObjNum 1550 snap 396 seq_num 1550) dirty exists 2014-07-01 17:13:19,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 1185 (ObjNum 1500 snap 369 seq_num 1500) dirty exists 2014-07-01 17:13:20,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3391: done (2 left) 2014-07-01 17:13:20,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3392: done (1 left) 2014-07-01 17:13:20,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3393: done (0 left) 2014-07-01 17:13:20,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3394: snap_create 2014-07-01 17:13:21,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:3394: done (0 left) 2014-07-01 17:13:21,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: write oid 155 current snap is 398 2014-07-01 17:13:21,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: seq_num 1551 ranges {671054=573720,1950785=520397,2940899=530155} 2014-07-01 17:13:21,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: writing mira0437250-155 from 671054 to 1244774 tid 1 2014-07-01 17:13:21,483.483 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: writing mira0437250-155 from 1950785 to 2471182 tid 2 2014-07-01 17:13:21,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: writing mira0437250-155 from 2940899 to 3471054 tid 3 2014-07-01 17:13:21,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:3396: rollback oid 121 current snap is 398 2014-07-01 17:13:21,513.513 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 121 to 371 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 1 to mira0437250-155 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 2 to mira0437250-155 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3397: rollback oid 61 current snap is 398 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 61 to 388 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: write oid 145 current snap is 398 2014-07-01 17:13:21,625.625 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: seq_num 1552 ranges {767308=545524,1781269=740992,3142124=25184} 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: writing mira0437250-145 from 767308 to 1312832 tid 1 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: writing mira0437250-145 from 1781269 to 2522261 tid 2 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: writing mira0437250-145 from 3142124 to 3167308 tid 3 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: read oid 189 snap 355 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 0 (ObjNum 1 snap 1 seq_num 16568976) dirty dne 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 0 (ObjNum 1 snap 1 seq_num 40516736) dirty dne 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 3 to mira0437250-155 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 5 to mira0437250-155 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: finishing write tid 6 to mira0437250-155 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 1228 (ObjNum 1551 snap 398 seq_num 1551) dirty exists 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: finishing write tid 1 to mira0437250-145 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3395: done (4 left) 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3396: done (3 left) 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3397: done (2 left) 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: write oid 454 current snap is 398 2014-07-01 17:13:21,626.626 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: seq_num 1553 ranges {462537=668752,1894376=562612,2862536=1} 2014-07-01 17:13:21,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: writing mira0437250-454 from 462537 to 1131289 tid 1 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: writing mira0437250-454 from 1894376 to 2456988 tid 2 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: writing mira0437250-454 from 2862536 to 2862537 tid 3 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: write oid 376 current snap is 398 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: seq_num 1554 ranges {655942=1} 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: writing mira0437250-376 from 655942 to 655943 tid 1 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3402: read oid 408 snap -1 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: finishing write tid 2 to mira0437250-145 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: finishing write tid 3 to mira0437250-145 2014-07-01 17:13:21,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: expect (ObjNum 943 snap 173 seq_num 943) 2014-07-01 17:13:22,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: finishing write tid 4 to mira0435601-484 2014-07-01 17:13:22,032.032 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 484 v 381 (ObjNum 2337 snap 0 seq_num 2337) dirty exists 2014-07-01 17:13:22,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 2 to mira0435601-418 2014-07-01 17:13:22,033.033 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 3 to mira0435601-418 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 4 to mira0435601-418 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 6 to mira0435601-418 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: done (4 left) 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3809: done (3 left) 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3810: done (2 left) 2014-07-01 17:13:22,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3813: read oid 353 snap -1 2014-07-01 17:13:22,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: finishing write tid 7 to mira0435601-418 2014-07-01 17:13:22,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 418 v 365 (ObjNum 2338 snap 0 seq_num 2338) dirty exists 2014-07-01 17:13:22,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3812: expect (ObjNum 1301 snap 0 seq_num 1301) 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: finishing write tid 5 to mira0437250-145 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: finishing write tid 6 to mira0437250-145 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 1001 (ObjNum 1552 snap 398 seq_num 1552) dirty exists 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: finishing write tid 1 to mira0437250-454 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: finishing write tid 2 to mira0437250-454 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3398: done (4 left) 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3399: done (3 left) 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: write oid 475 current snap is 398 2014-07-01 17:13:22,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: seq_num 1555 ranges {352196=1} 2014-07-01 17:13:22,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: writing mira0437250-475 from 352196 to 352197 tid 1 2014-07-01 17:13:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3404: delete oid 37 current snap is 398 2014-07-01 17:13:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: finishing write tid 3 to mira0437250-454 2014-07-01 17:13:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: finishing write tid 5 to mira0437250-454 2014-07-01 17:13:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: finishing write tid 6 to mira0437250-454 2014-07-01 17:13:22,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 1233 (ObjNum 1553 snap 398 seq_num 1553) dirty exists 2014-07-01 17:13:22,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 1 to mira0437250-376 2014-07-01 17:13:22,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 3 to mira0437250-376 2014-07-01 17:13:22,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: finishing write tid 4 to mira0437250-376 2014-07-01 17:13:22,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 1716 (ObjNum 1554 snap 398 seq_num 1554) dirty exists 2014-07-01 17:13:22,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3402: expect (ObjNum 407 snap 0 seq_num 407) 2014-07-01 17:13:22,812.812 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:13:22,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: finishing write tid 1 to mira0437250-475 2014-07-01 17:13:22,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3400: done (4 left) 2014-07-01 17:13:22,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3401: done (3 left) 2014-07-01 17:13:22,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3402: done (2 left) 2014-07-01 17:13:22,813.813 INFO:teuthology.task.rados.rados.0.mira043.stdout:3404: done (1 left) 2014-07-01 17:13:22,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: write oid 355 current snap is 398 2014-07-01 17:13:22,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: seq_num 1557 ranges {757184=743490,2128314=228870} 2014-07-01 17:13:22,846.846 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: writing mira0437250-355 from 757184 to 1500674 tid 1 2014-07-01 17:13:22,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: writing mira0437250-355 from 2128314 to 2357184 tid 2 2014-07-01 17:13:22,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3406: read oid 88 snap -1 2014-07-01 17:13:22,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: finishing write tid 3 to mira0437250-475 2014-07-01 17:13:22,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: finishing write tid 4 to mira0437250-475 2014-07-01 17:13:22,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 1268 (ObjNum 1555 snap 398 seq_num 1555) dirty exists 2014-07-01 17:13:22,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: finishing write tid 1 to mira0437250-355 2014-07-01 17:13:22,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:3403: done (2 left) 2014-07-01 17:13:22,973.973 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: snap_create 2014-07-01 17:13:22,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3406: expect (ObjNum 1513 snap 376 seq_num 1513) 2014-07-01 17:13:22,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: finishing write tid 2 to mira0437250-355 2014-07-01 17:13:22,974.974 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: finishing write tid 4 to mira0437250-355 2014-07-01 17:13:22,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: finishing write tid 5 to mira0437250-355 2014-07-01 17:13:22,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 355 v 1409 (ObjNum 1557 snap 398 seq_num 1557) dirty exists 2014-07-01 17:13:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3405: done (2 left) 2014-07-01 17:13:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3406: done (1 left) 2014-07-01 17:13:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3407: done (0 left) 2014-07-01 17:13:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: write oid 433 current snap is 399 2014-07-01 17:13:23,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: seq_num 1558 ranges {451390=716318,1841422=209968} 2014-07-01 17:13:23,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: writing mira0437250-433 from 451390 to 1167708 tid 1 2014-07-01 17:13:23,493.493 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 14, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.008213', u'skew': u'0.001456', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.004369', u'skew': u'0.001461', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:12:45.735760', u'name': u'c', u'avail_percent': 92, u'kb_total': 944797076, u'kb_avail': 878457832, u'health': u'HEALTH_OK', u'kb_used': 18323220, u'store_stats': {u'bytes_total': 5502633, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4978329}}, {u'last_updated': u'2014-07-01 08:12:56.013682', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882997112, u'health': u'HEALTH_OK', u'kb_used': 13783940, u'store_stats': {u'bytes_total': 8122905, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4977161}}, {u'last_updated': u'2014-07-01 08:13:13.317839', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882949744, u'health': u'HEALTH_OK', u'kb_used': 13831308, u'store_stats': {u'bytes_total': 11268635, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 4977163}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'2 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 64 > pgp_num 54'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_1 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 534, u'data_bytes': 2483010846, u'bytes_used': 5836234752, u'op_per_sec': 29, u'version': 3000, u'pgs_by_state': [{u'count': 534, u'state_name': u'active+clean'}], u'read_bytes_sec': 171051, u'write_bytes_sec': 6053194, u'bytes_avail': 5992462811136}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1484, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:13:23,494.494 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:13:23,494.494 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:13:23,494.494 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_1 2014-07-01 17:13:23,494.494 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:23,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: writing mira0437250-433 from 1841422 to 2051390 tid 2 2014-07-01 17:13:23,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: snap_create 2014-07-01 17:13:23,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: finishing write tid 1 to mira0437250-433 2014-07-01 17:13:23,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: finishing write tid 2 to mira0437250-433 2014-07-01 17:13:23,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: finishing write tid 4 to mira0437250-433 2014-07-01 17:13:23,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: finishing write tid 5 to mira0437250-433 2014-07-01 17:13:23,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 773 (ObjNum 1558 snap 399 seq_num 1558) dirty exists 2014-07-01 17:13:24,301.301 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:24,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3813: expect (ObjNum 1767 snap 0 seq_num 1767) 2014-07-01 17:13:24,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3408: done (1 left) 2014-07-01 17:13:24,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3409: done (0 left) 2014-07-01 17:13:24,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: read oid 280 snap -1 2014-07-01 17:13:24,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: snap_remove snap 391 2014-07-01 17:13:24,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: expect (ObjNum 989 snap 187 seq_num 989) 2014-07-01 17:13:25,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3410: done (1 left) 2014-07-01 17:13:25,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3411: done (0 left) 2014-07-01 17:13:25,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3412: rollback oid 173 current snap is 400 2014-07-01 17:13:25,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 173 to 304 2014-07-01 17:13:25,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: snap_create 2014-07-01 17:13:25,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 173 v 1780 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:13:26,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:3412: done (1 left) 2014-07-01 17:13:26,470.470 INFO:teuthology.task.rados.rados.0.mira043.stdout:3413: done (0 left) 2014-07-01 17:13:26,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3414: snap_create 2014-07-01 17:13:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: done (2 left) 2014-07-01 17:13:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3812: done (1 left) 2014-07-01 17:13:27,397.397 INFO:teuthology.task.rados.rados.0.mira043.stdout:3813: done (0 left) 2014-07-01 17:13:27,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 417 current snap is 0 2014-07-01 17:13:27,399.399 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: seq_num 2339 ranges {692224=655360,1347584=774144,2121728=430080} 2014-07-01 17:13:27,424.424 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: writing mira0435601-417 from 692224 to 1347584 tid 1 2014-07-01 17:13:27,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: writing mira0435601-417 from 1347584 to 2121728 tid 2 2014-07-01 17:13:27,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: writing mira0435601-417 from 2121728 to 2551808 tid 3 2014-07-01 17:13:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 394 current snap is 0 2014-07-01 17:13:27,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: seq_num 2340 ranges {10645504=172032} 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: writing mira0435601-394 from 10645504 to 10817536 tid 1 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: read oid 109 snap -1 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: finishing write tid 1 to mira0435601-417 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: finishing write tid 2 to mira0435601-417 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: finishing write tid 3 to mira0435601-417 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: finishing write tid 5 to mira0435601-417 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: finishing write tid 6 to mira0435601-417 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3414: done (0 left) 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3415: delete oid 142 current snap is 402 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3415: done (0 left) 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: write oid 149 current snap is 402 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: seq_num 1560 ranges {57401=1} 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: writing mira0437250-149 from 57401 to 57402 tid 1 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: write oid 303 current snap is 402 2014-07-01 17:13:27,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: seq_num 1561 ranges {552631=419251,1748746=510669,2933953=579076,3752630=1} 2014-07-01 17:13:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: writing mira0437250-303 from 552631 to 971882 tid 1 2014-07-01 17:13:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: writing mira0437250-303 from 1748746 to 2259415 tid 2 2014-07-01 17:13:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 163 (ObjNum 2339 snap 0 seq_num 2339) dirty exists 2014-07-01 17:13:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: done (2 left) 2014-07-01 17:13:27,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 311 current snap is 0 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: seq_num 2341 ranges {9650176=602112,10252288=786432,11038720=16384} 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: writing mira0437250-303 from 2933953 to 3513029 tid 3 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: writing mira0437250-303 from 3752630 to 3752631 tid 4 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: read oid 208 snap -1 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: finishing write tid 1 to mira0437250-149 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 1 to mira0437250-303 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: finishing write tid 3 to mira0437250-149 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: finishing write tid 4 to mira0437250-149 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 149 v 1773 (ObjNum 1560 snap 402 seq_num 1560) dirty exists 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 2 to mira0437250-303 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3416: done (2 left) 2014-07-01 17:13:27,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: write oid 445 current snap is 402 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: seq_num 1562 ranges {497460=1} 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: writing mira0437250-445 from 497460 to 497461 tid 1 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3420: read oid 417 snap -1 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: read oid 356 snap -1 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: snap_remove snap 386 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: finishing write tid 1 to mira0437250-445 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: finishing write tid 3 to mira0437250-445 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: finishing write tid 4 to mira0437250-445 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 1490 (ObjNum 1562 snap 402 seq_num 1562) dirty exists 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 3 to mira0437250-303 2014-07-01 17:13:27,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 4 to mira0437250-303 2014-07-01 17:13:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 6 to mira0437250-303 2014-07-01 17:13:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: finishing write tid 7 to mira0437250-303 2014-07-01 17:13:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 303 v 1786 (ObjNum 1561 snap 402 seq_num 1561) dirty exists 2014-07-01 17:13:27,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3420: expect (ObjNum 1288 snap 292 seq_num 1288) 2014-07-01 17:13:27,664.664 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pgp_num 86' 2014-07-01 17:13:27,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: writing mira0435601-311 from 9650176 to 10252288 tid 1 2014-07-01 17:13:27,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: writing mira0435601-311 from 10252288 to 11038720 tid 2 2014-07-01 17:13:27,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: writing mira0435601-311 from 11038720 to 11055104 tid 3 2014-07-01 17:13:27,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 496 current snap is 0 2014-07-01 17:13:27,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: seq_num 2342 ranges {5005312=475136,5480448=770048,6250496=32768} 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0435601-496 from 5005312 to 5480448 tid 1 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0435601-496 from 5480448 to 6250496 tid 2 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0435601-496 from 6250496 to 6283264 tid 3 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3819: delete oid 441 current snap is 0 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: finishing write tid 1 to mira0435601-394 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: finishing write tid 3 to mira0435601-394 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: finishing write tid 4 to mira0435601-394 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 215 (ObjNum 2340 snap 0 seq_num 2340) dirty exists 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 1 to mira0435601-311 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 2 to mira0435601-311 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 3 to mira0435601-311 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 5 to mira0435601-311 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 6 to mira0435601-311 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 295 (ObjNum 2341 snap 0 seq_num 2341) dirty exists 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 1 to mira0435601-496 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 2 to mira0435601-496 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: done (4 left) 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: done (3 left) 2014-07-01 17:13:27,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: done (2 left) 2014-07-01 17:13:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3819: done (1 left) 2014-07-01 17:13:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: read oid 34 snap -1 2014-07-01 17:13:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 3 to mira0435601-496 2014-07-01 17:13:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 79 current snap is 0 2014-07-01 17:13:27,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: seq_num 2344 ranges {2797568=774144,3571712=585728,4157440=774144,4931584=245760} 2014-07-01 17:13:27,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0435601-79 from 2797568 to 3571712 tid 1 2014-07-01 17:13:27,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0435601-79 from 3571712 to 4157440 tid 2 2014-07-01 17:13:27,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0435601-79 from 4157440 to 4931584 tid 3 2014-07-01 17:13:27,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0435601-79 from 4931584 to 5177344 tid 4 2014-07-01 17:13:27,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 230 current snap is 0 2014-07-01 17:13:27,832.832 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: seq_num 2345 ranges {2252800=696320} 2014-07-01 17:13:27,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: writing mira0435601-230 from 2252800 to 2949120 tid 1 2014-07-01 17:13:27,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3823: read oid 130 snap -1 2014-07-01 17:13:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 5 to mira0435601-496 2014-07-01 17:13:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 6 to mira0435601-496 2014-07-01 17:13:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 496 v 198 (ObjNum 2342 snap 0 seq_num 2342) dirty exists 2014-07-01 17:13:27,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: expect (ObjNum 2053 snap 0 seq_num 2053) 2014-07-01 17:13:28,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: expect (ObjNum 749 snap 109 seq_num 749) 2014-07-01 17:13:28,512.512 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pgp_num to 86 2014-07-01 17:13:28,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3417: done (5 left) 2014-07-01 17:13:28,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3418: done (4 left) 2014-07-01 17:13:28,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3419: done (3 left) 2014-07-01 17:13:28,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3420: done (2 left) 2014-07-01 17:13:28,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:3421: done (1 left) 2014-07-01 17:13:28,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:3422: done (0 left) 2014-07-01 17:13:28,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: rollback oid 180 current snap is 402 2014-07-01 17:13:28,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 180 to 355 2014-07-01 17:13:28,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3424: snap_remove snap 395 2014-07-01 17:13:28,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 180 v 1600 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:13:29,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3423: done (1 left) 2014-07-01 17:13:29,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3424: done (0 left) 2014-07-01 17:13:29,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3425: delete oid 332 current snap is 402 2014-07-01 17:13:29,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3425: done (0 left) 2014-07-01 17:13:29,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: snap_remove snap 390 2014-07-01 17:13:30,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3426: done (0 left) 2014-07-01 17:13:30,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: read oid 48 snap -1 2014-07-01 17:13:30,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: delete oid 243 current snap is 402 2014-07-01 17:13:30,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3428: done (1 left) 2014-07-01 17:13:30,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: snap_create 2014-07-01 17:13:30,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: expect (ObjNum 47 snap 0 seq_num 47) 2014-07-01 17:13:31,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 1 to mira0435601-79 2014-07-01 17:13:31,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 2 to mira0435601-79 2014-07-01 17:13:31,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 3 to mira0435601-79 2014-07-01 17:13:31,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 4 to mira0435601-79 2014-07-01 17:13:31,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 6 to mira0435601-79 2014-07-01 17:13:31,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 7 to mira0435601-79 2014-07-01 17:13:31,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 79 v 288 (ObjNum 2344 snap 0 seq_num 2344) dirty exists 2014-07-01 17:13:31,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 1 to mira0435601-230 2014-07-01 17:13:31,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 3 to mira0435601-230 2014-07-01 17:13:31,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 4 to mira0435601-230 2014-07-01 17:13:31,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 230 v 357 (ObjNum 2345 snap 0 seq_num 2345) dirty exists 2014-07-01 17:13:31,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3823: expect (ObjNum 1942 snap 0 seq_num 1942) 2014-07-01 17:13:31,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:3427: done (1 left) 2014-07-01 17:13:31,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3429: done (0 left) 2014-07-01 17:13:31,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: write oid 264 current snap is 403 2014-07-01 17:13:31,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: seq_num 1565 ranges {598702=742564,1982682=216020} 2014-07-01 17:13:31,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: writing mira0437250-264 from 598702 to 1341266 tid 1 2014-07-01 17:13:31,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: writing mira0437250-264 from 1982682 to 2198702 tid 2 2014-07-01 17:13:31,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: read oid 288 snap -1 2014-07-01 17:13:31,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: snap_remove snap 396 2014-07-01 17:13:31,953.953 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: expect (ObjNum 755 snap 110 seq_num 755) 2014-07-01 17:13:32,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: finishing write tid 1 to mira0437250-264 2014-07-01 17:13:32,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: finishing write tid 2 to mira0437250-264 2014-07-01 17:13:32,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: finishing write tid 4 to mira0437250-264 2014-07-01 17:13:32,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: finishing write tid 5 to mira0437250-264 2014-07-01 17:13:32,158.158 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 264 v 1720 (ObjNum 1565 snap 403 seq_num 1565) dirty exists 2014-07-01 17:13:32,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3430: done (2 left) 2014-07-01 17:13:32,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3431: done (1 left) 2014-07-01 17:13:32,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:3432: done (0 left) 2014-07-01 17:13:32,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: read oid 32 snap 385 2014-07-01 17:13:32,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:3434: rollback oid 71 current snap is 403 2014-07-01 17:13:32,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 71 to 373 2014-07-01 17:13:32,533.533 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: snap_remove snap 388 2014-07-01 17:13:32,543.543 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 71 v 0 (ObjNum 34780240 snap 0 seq_num 32) dirty dne 2014-07-01 17:13:33,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: done (4 left) 2014-07-01 17:13:33,101.101 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: done (3 left) 2014-07-01 17:13:33,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: done (2 left) 2014-07-01 17:13:33,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: done (1 left) 2014-07-01 17:13:33,102.102 INFO:teuthology.task.rados.rados.0.mira043.stdout:3823: done (0 left) 2014-07-01 17:13:33,103.103 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 143 current snap is 0 2014-07-01 17:13:33,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: seq_num 2346 ranges {3047424=720896,3768320=749568,4517888=454656} 2014-07-01 17:13:33,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: writing mira0435601-143 from 3047424 to 3768320 tid 1 2014-07-01 17:13:33,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: writing mira0435601-143 from 3768320 to 4517888 tid 2 2014-07-01 17:13:33,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: writing mira0435601-143 from 4517888 to 4972544 tid 3 2014-07-01 17:13:33,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: read oid 56 snap -1 2014-07-01 17:13:33,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: finishing write tid 1 to mira0435601-143 2014-07-01 17:13:33,269.269 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 481 current snap is 0 2014-07-01 17:13:33,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: seq_num 2347 ranges {4354048=593920,4947968=577536,5525504=593920,6119424=32768} 2014-07-01 17:13:33,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: writing mira0435601-481 from 4354048 to 4947968 tid 1 2014-07-01 17:13:33,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: writing mira0435601-481 from 4947968 to 5525504 tid 2 2014-07-01 17:13:33,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: writing mira0435601-481 from 5525504 to 6119424 tid 3 2014-07-01 17:13:33,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: writing mira0435601-481 from 6119424 to 6152192 tid 4 2014-07-01 17:13:33,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 299 current snap is 0 2014-07-01 17:13:33,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: seq_num 2348 ranges {5509120=466944,5976064=565248,6541312=421888,6963200=217088} 2014-07-01 17:13:33,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: writing mira0435601-299 from 5509120 to 5976064 tid 1 2014-07-01 17:13:33,315.315 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: writing mira0435601-299 from 5976064 to 6541312 tid 2 2014-07-01 17:13:33,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: writing mira0435601-299 from 6541312 to 6963200 tid 3 2014-07-01 17:13:33,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: writing mira0435601-299 from 6963200 to 7180288 tid 4 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 323 current snap is 0 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: seq_num 2349 ranges {1642496=540672,2183168=794624,2977792=8192} 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: writing mira0435601-323 from 1642496 to 2183168 tid 1 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: writing mira0435601-323 from 2183168 to 2977792 tid 2 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: writing mira0435601-323 from 2977792 to 2985984 tid 3 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 281 current snap is 0 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: seq_num 2350 ranges {0=413696,413696=729088,1142784=73728} 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: writing mira0435601-281 from 0 to 413696 tid 1 2014-07-01 17:13:33,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: writing mira0435601-281 from 413696 to 1142784 tid 2 2014-07-01 17:13:33,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: writing mira0435601-281 from 1142784 to 1216512 tid 3 2014-07-01 17:13:33,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 335 current snap is 0 2014-07-01 17:13:33,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: seq_num 2351 ranges {0=688128,688128=401408} 2014-07-01 17:13:33,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: writing mira0435601-335 from 0 to 688128 tid 1 2014-07-01 17:13:33,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: writing mira0435601-335 from 688128 to 1089536 tid 2 2014-07-01 17:13:33,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 324 current snap is 0 2014-07-01 17:13:33,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: seq_num 2352 ranges {0=712704,712704=790528,1503232=557056,2060288=258048} 2014-07-01 17:13:33,466.466 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0435601-324 from 0 to 712704 tid 1 2014-07-01 17:13:33,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0435601-324 from 712704 to 1503232 tid 2 2014-07-01 17:13:33,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0435601-324 from 1503232 to 2060288 tid 3 2014-07-01 17:13:33,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0435601-324 from 2060288 to 2318336 tid 4 2014-07-01 17:13:33,528.528 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:13:33,528.528 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:13:33,529.529 INFO:teuthology.task.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2014-07-01 17:13:33,529.529 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 1 0' 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 403 current snap is 0 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: seq_num 2353 ranges {12288=585728,598016=618496,1216512=786432,2002944=200704} 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: writing mira0435601-403 from 12288 to 598016 tid 1 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: writing mira0435601-403 from 598016 to 1216512 tid 2 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: writing mira0435601-403 from 1216512 to 2002944 tid 3 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: writing mira0435601-403 from 2002944 to 2203648 tid 4 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 300 current snap is 0 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: seq_num 2354 ranges {4214784=503808} 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3433: done (2 left) 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3434: done (1 left) 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3435: done (0 left) 2014-07-01 17:13:33,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: write oid 163 current snap is 403 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: seq_num 1566 ranges {692908=714369,1892907=1} 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: writing mira0437250-163 from 692908 to 1407277 tid 1 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: writing mira0437250-163 from 1892907 to 1892908 tid 2 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3437: read oid 38 snap 368 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: rollback oid 305 current snap is 403 2014-07-01 17:13:33,617.617 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 305 to 375 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3439: rollback oid 77 current snap is 403 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 77 to 401 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: write oid 86 current snap is 403 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: seq_num 1567 ranges {637738=438621,1774851=717312,2637737=1} 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0437250-86 from 637738 to 1076359 tid 1 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0437250-86 from 1774851 to 2492163 tid 2 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: writing mira0437250-86 from 2637737 to 2637738 tid 3 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3441: delete oid 214 current snap is 403 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: writing mira0435601-300 from 4214784 to 4718592 tid 1 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3834: read oid 320 snap -1 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: finishing write tid 2 to mira0435601-143 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: finishing write tid 3 to mira0435601-143 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: finishing write tid 5 to mira0435601-143 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: finishing write tid 6 to mira0435601-143 2014-07-01 17:13:33,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 143 v 173 (ObjNum 2346 snap 0 seq_num 2346) dirty exists 2014-07-01 17:13:33,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: expect (ObjNum 2119 snap 0 seq_num 2119) 2014-07-01 17:13:33,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 77 v 0 (ObjNum 0 snap 0 seq_num 32) dirty dne 2014-07-01 17:13:33,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 1356 (ObjNum 1234 snap 269 seq_num 1234) dirty exists 2014-07-01 17:13:33,669.669 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: finishing write tid 1 to mira0437250-163 2014-07-01 17:13:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: finishing write tid 2 to mira0437250-163 2014-07-01 17:13:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: finishing write tid 4 to mira0437250-163 2014-07-01 17:13:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: finishing write tid 5 to mira0437250-163 2014-07-01 17:13:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 163 v 1633 (ObjNum 1566 snap 403 seq_num 1566) dirty exists 2014-07-01 17:13:33,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 1 to mira0437250-86 2014-07-01 17:13:33,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 2 to mira0437250-86 2014-07-01 17:13:33,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 3 to mira0437250-86 2014-07-01 17:13:33,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 5 to mira0437250-86 2014-07-01 17:13:33,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: finishing write tid 6 to mira0437250-86 2014-07-01 17:13:33,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 86 v 626 (ObjNum 1567 snap 403 seq_num 1567) dirty exists 2014-07-01 17:13:33,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3436: done (5 left) 2014-07-01 17:13:33,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3437: done (4 left) 2014-07-01 17:13:33,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3438: done (3 left) 2014-07-01 17:13:33,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3439: done (2 left) 2014-07-01 17:13:33,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3440: done (1 left) 2014-07-01 17:13:33,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3441: done (0 left) 2014-07-01 17:13:33,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3442: rollback oid 343 current snap is 403 2014-07-01 17:13:33,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 343 to 361 2014-07-01 17:13:33,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: read oid 37 snap 384 2014-07-01 17:13:33,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3444: read oid 358 snap -1 2014-07-01 17:13:33,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: read oid 313 snap 368 2014-07-01 17:13:33,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3446: rollback oid 91 current snap is 403 2014-07-01 17:13:33,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 91 to 379 2014-07-01 17:13:33,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: read oid 492 snap 392 2014-07-01 17:13:33,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: write oid 26 current snap is 403 2014-07-01 17:13:33,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: seq_num 1569 ranges {794149=418326,1816619=609127,2854412=739737} 2014-07-01 17:13:33,675.675 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: writing mira0437250-26 from 794149 to 1212475 tid 1 2014-07-01 17:13:33,678.678 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: writing mira0437250-26 from 1816619 to 2425746 tid 2 2014-07-01 17:13:33,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: writing mira0437250-26 from 2854412 to 3594149 tid 3 2014-07-01 17:13:33,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: write oid 157 current snap is 403 2014-07-01 17:13:33,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: seq_num 1570 ranges {338978=1} 2014-07-01 17:13:33,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: writing mira0437250-157 from 338978 to 338979 tid 1 2014-07-01 17:13:33,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: write oid 75 current snap is 403 2014-07-01 17:13:33,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: seq_num 1571 ranges {682785=400000} 2014-07-01 17:13:33,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: writing mira0437250-75 from 682785 to 1082785 tid 1 2014-07-01 17:13:33,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3451: delete oid 280 current snap is 403 2014-07-01 17:13:33,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 1502 (ObjNum 645 snap 67 seq_num 645) dirty exists 2014-07-01 17:13:33,783.783 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: expect (ObjNum 1531 snap 384 seq_num 1531) 2014-07-01 17:13:34,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 91 v 1504 (ObjNum 1461 snap 355 seq_num 1461) dirty exists 2014-07-01 17:13:34,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: finishing write tid 1 to mira0437250-26 2014-07-01 17:13:34,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: finishing write tid 2 to mira0437250-26 2014-07-01 17:13:34,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: finishing write tid 3 to mira0437250-26 2014-07-01 17:13:34,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: finishing write tid 5 to mira0437250-26 2014-07-01 17:13:34,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3442: done (9 left) 2014-07-01 17:13:34,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3444: done (8 left) 2014-07-01 17:13:34,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3446: done (7 left) 2014-07-01 17:13:34,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3447: done (6 left) 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3451: done (5 left) 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3452: snap_remove snap 399 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: finishing write tid 6 to mira0437250-26 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 1116 (ObjNum 1569 snap 403 seq_num 1569) dirty exists 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: finishing write tid 1 to mira0437250-157 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: finishing write tid 3 to mira0437250-157 2014-07-01 17:13:34,503.503 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: expect (ObjNum 1168 snap 246 seq_num 1168) 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: finishing write tid 4 to mira0437250-157 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 157 v 1727 (ObjNum 1570 snap 403 seq_num 1570) dirty exists 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: finishing write tid 1 to mira0437250-75 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: finishing write tid 3 to mira0437250-75 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: finishing write tid 4 to mira0437250-75 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 949 (ObjNum 1571 snap 403 seq_num 1571) dirty exists 2014-07-01 17:13:34,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: expect (ObjNum 1087 snap 218 seq_num 1087) 2014-07-01 17:13:34,579.579 INFO:teuthology.orchestra.run.mira079.stderr:set osd.1 primary-affinity to 0 (802) 2014-07-01 17:13:36,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 1 to mira0435601-481 2014-07-01 17:13:36,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 2 to mira0435601-481 2014-07-01 17:13:36,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 3 to mira0435601-481 2014-07-01 17:13:36,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 4 to mira0435601-481 2014-07-01 17:13:36,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 6 to mira0435601-481 2014-07-01 17:13:36,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: done (10 left) 2014-07-01 17:13:36,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: done (9 left) 2014-07-01 17:13:36,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 133 current snap is 0 2014-07-01 17:13:36,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: seq_num 2355 ranges {3780608=532480,4313088=786432,5099520=417792} 2014-07-01 17:13:36,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0435601-133 from 3780608 to 4313088 tid 1 2014-07-01 17:13:36,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0435601-133 from 4313088 to 5099520 tid 2 2014-07-01 17:13:36,937.937 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0435601-133 from 5099520 to 5517312 tid 3 2014-07-01 17:13:37,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 155 current snap is 0 2014-07-01 17:13:37,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: seq_num 2356 ranges {2129920=561152} 2014-07-01 17:13:37,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: writing mira0435601-155 from 2129920 to 2691072 tid 1 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3837: read oid 497 snap -1 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 7 to mira0435601-481 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 172 (ObjNum 2347 snap 0 seq_num 2347) dirty exists 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 1 to mira0435601-299 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 2 to mira0435601-299 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 3 to mira0435601-299 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 4 to mira0435601-299 2014-07-01 17:13:37,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 6 to mira0435601-299 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: finishing write tid 7 to mira0435601-299 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 299 v 330 (ObjNum 2348 snap 0 seq_num 2348) dirty exists 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 1 to mira0435601-323 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 2 to mira0435601-323 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 3 to mira0435601-323 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 5 to mira0435601-323 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: finishing write tid 1 to mira0435601-281 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 6 to mira0435601-323 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 219 (ObjNum 2349 snap 0 seq_num 2349) dirty exists 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: finishing write tid 2 to mira0435601-281 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: finishing write tid 3 to mira0435601-281 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: done (11 left) 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: done (10 left) 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: done (9 left) 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 351 current snap is 0 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: seq_num 2357 ranges {7430144=675840,8105984=602112,8708096=200704} 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3443: done (5 left) 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3445: done (4 left) 2014-07-01 17:13:37,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3448: done (3 left) 2014-07-01 17:13:37,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3449: done (2 left) 2014-07-01 17:13:37,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3450: done (1 left) 2014-07-01 17:13:37,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3452: done (0 left) 2014-07-01 17:13:37,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: write oid 195 current snap is 403 2014-07-01 17:13:37,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: seq_num 1573 ranges {722844=741639,1522843=1} 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: writing mira0435601-351 from 7430144 to 8105984 tid 1 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: writing mira0435601-351 from 8105984 to 8708096 tid 2 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: writing mira0435601-351 from 8708096 to 8908800 tid 3 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 421 current snap is 0 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: seq_num 2358 ranges {0=753664,753664=401408} 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: writing mira0437250-195 from 722844 to 1464483 tid 1 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: writing mira0437250-195 from 1522843 to 1522844 tid 2 2014-07-01 17:13:37,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3454: snap_create 2014-07-01 17:13:37,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 1 to mira0437250-195 2014-07-01 17:13:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 2 to mira0437250-195 2014-07-01 17:13:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 4 to mira0437250-195 2014-07-01 17:13:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: finishing write tid 5 to mira0437250-195 2014-07-01 17:13:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 195 v 789 (ObjNum 1573 snap 403 seq_num 1573) dirty exists 2014-07-01 17:13:37,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: writing mira0435601-421 from 0 to 753664 tid 1 2014-07-01 17:13:37,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: writing mira0435601-421 from 753664 to 1155072 tid 2 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 109 current snap is 0 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: seq_num 2359 ranges {0=782336,782336=765952,1548288=733184,2281472=106496} 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: writing mira0435601-109 from 0 to 782336 tid 1 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: writing mira0435601-109 from 782336 to 1548288 tid 2 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: writing mira0435601-109 from 1548288 to 2281472 tid 3 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: writing mira0435601-109 from 2281472 to 2387968 tid 4 2014-07-01 17:13:37,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 23 current snap is 0 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: seq_num 2360 ranges {2973696=495616,3469312=401408} 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3453: done (1 left) 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3454: done (0 left) 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: write oid 133 current snap is 404 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: seq_num 1574 ranges {783002=713444,2011217=710503,2783001=1} 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0437250-133 from 783002 to 1496446 tid 1 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0437250-133 from 2011217 to 2721720 tid 2 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: writing mira0437250-133 from 2783001 to 2783002 tid 3 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: snap_create 2014-07-01 17:13:37,224.224 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 1 to mira0437250-133 2014-07-01 17:13:37,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: writing mira0435601-23 from 2973696 to 3469312 tid 1 2014-07-01 17:13:37,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: writing mira0435601-23 from 3469312 to 3870720 tid 2 2014-07-01 17:13:37,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 154 current snap is 0 2014-07-01 17:13:37,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: seq_num 2361 ranges {0=524288,524288=737280,1261568=679936,1941504=188416} 2014-07-01 17:13:37,275.275 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: writing mira0435601-154 from 0 to 524288 tid 1 2014-07-01 17:13:37,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: writing mira0435601-154 from 524288 to 1261568 tid 2 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: writing mira0435601-154 from 1261568 to 1941504 tid 3 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: writing mira0435601-154 from 1941504 to 2129920 tid 4 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: read oid 260 snap -1 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3844: read oid 266 snap -1 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: finishing write tid 5 to mira0435601-281 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: finishing write tid 6 to mira0435601-281 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 281 v 219 (ObjNum 2350 snap 0 seq_num 2350) dirty exists 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 1 to mira0435601-335 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 2 to mira0435601-335 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 4 to mira0435601-335 2014-07-01 17:13:37,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 5 to mira0435601-335 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 252 (ObjNum 2351 snap 0 seq_num 2351) dirty exists 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 1 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 2 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 3 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 4 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 6 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 7 to mira0435601-324 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 324 v 300 (ObjNum 2352 snap 0 seq_num 2352) dirty exists 2014-07-01 17:13:37,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 1 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 2 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 3 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 4 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 6 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: finishing write tid 7 to mira0435601-403 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 391 (ObjNum 2353 snap 0 seq_num 2353) dirty exists 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 1 to mira0435601-300 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 3 to mira0435601-300 2014-07-01 17:13:37,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3834: expect (ObjNum 952 snap 0 seq_num 952) 2014-07-01 17:13:37,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 2 to mira0437250-133 2014-07-01 17:13:37,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 3 to mira0437250-133 2014-07-01 17:13:37,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 5 to mira0437250-133 2014-07-01 17:13:37,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: finishing write tid 6 to mira0437250-133 2014-07-01 17:13:37,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 837 (ObjNum 1574 snap 404 seq_num 1574) dirty exists 2014-07-01 17:13:38,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3455: done (1 left) 2014-07-01 17:13:38,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3456: done (0 left) 2014-07-01 17:13:38,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: write oid 16 current snap is 405 2014-07-01 17:13:38,130.130 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: seq_num 1575 ranges {479256=437696,1326373=780698,2479255=1} 2014-07-01 17:13:38,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0437250-16 from 479256 to 916952 tid 1 2014-07-01 17:13:38,167.167 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0437250-16 from 1326373 to 2107071 tid 2 2014-07-01 17:13:38,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: writing mira0437250-16 from 2479255 to 2479256 tid 3 2014-07-01 17:13:38,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3458: read oid 230 snap -1 2014-07-01 17:13:38,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 1 to mira0437250-16 2014-07-01 17:13:38,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: write oid 379 current snap is 405 2014-07-01 17:13:38,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: seq_num 1576 ranges {574485=693148,1723128=451357} 2014-07-01 17:13:38,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: writing mira0437250-379 from 574485 to 1267633 tid 1 2014-07-01 17:13:38,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: writing mira0437250-379 from 1723128 to 2174485 tid 2 2014-07-01 17:13:38,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: snap_create 2014-07-01 17:13:38,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 2 to mira0437250-16 2014-07-01 17:13:38,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 3 to mira0437250-16 2014-07-01 17:13:38,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 5 to mira0437250-16 2014-07-01 17:13:38,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: finishing write tid 6 to mira0437250-16 2014-07-01 17:13:38,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 16 v 1032 (ObjNum 1575 snap 405 seq_num 1575) dirty exists 2014-07-01 17:13:38,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: finishing write tid 1 to mira0437250-379 2014-07-01 17:13:38,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: finishing write tid 2 to mira0437250-379 2014-07-01 17:13:38,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: finishing write tid 4 to mira0437250-379 2014-07-01 17:13:38,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: finishing write tid 5 to mira0437250-379 2014-07-01 17:13:38,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 379 v 629 (ObjNum 1576 snap 405 seq_num 1576) dirty exists 2014-07-01 17:13:38,302.302 INFO:teuthology.task.rados.rados.0.mira043.stdout:3458: expect (ObjNum 692 snap 88 seq_num 692) 2014-07-01 17:13:38,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 4 to mira0435601-300 2014-07-01 17:13:38,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 300 v 221 (ObjNum 2354 snap 0 seq_num 2354) dirty exists 2014-07-01 17:13:38,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 1 to mira0435601-133 2014-07-01 17:13:38,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 2 to mira0435601-133 2014-07-01 17:13:38,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 3 to mira0435601-133 2014-07-01 17:13:38,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 5 to mira0435601-133 2014-07-01 17:13:38,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 6 to mira0435601-133 2014-07-01 17:13:38,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 133 v 242 (ObjNum 2355 snap 0 seq_num 2355) dirty exists 2014-07-01 17:13:38,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: finishing write tid 1 to mira0435601-155 2014-07-01 17:13:38,930.930 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: finishing write tid 3 to mira0435601-155 2014-07-01 17:13:38,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: finishing write tid 4 to mira0435601-155 2014-07-01 17:13:38,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 155 v 332 (ObjNum 2356 snap 0 seq_num 2356) dirty exists 2014-07-01 17:13:38,931.931 INFO:teuthology.task.rados.rados.0.mira043.stdout:3837: expect (ObjNum 1765 snap 0 seq_num 1765) 2014-07-01 17:13:39,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3457: done (3 left) 2014-07-01 17:13:39,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3458: done (2 left) 2014-07-01 17:13:39,335.335 INFO:teuthology.task.rados.rados.0.mira043.stdout:3459: done (1 left) 2014-07-01 17:13:39,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3460: done (0 left) 2014-07-01 17:13:39,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: write oid 387 current snap is 406 2014-07-01 17:13:39,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: seq_num 1577 ranges {767891=416377,1802189=707584,3224792=475244,3967890=1} 2014-07-01 17:13:39,349.349 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: writing mira0437250-387 from 767891 to 1184268 tid 1 2014-07-01 17:13:39,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: writing mira0437250-387 from 1802189 to 2509773 tid 2 2014-07-01 17:13:39,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: writing mira0437250-387 from 3224792 to 3700036 tid 3 2014-07-01 17:13:39,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: writing mira0437250-387 from 3967890 to 3967891 tid 4 2014-07-01 17:13:39,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3462: delete oid 142 current snap is 406 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 1 to mira0437250-387 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 2 to mira0437250-387 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 3 to mira0437250-387 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3462: done (1 left) 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: snap_create 2014-07-01 17:13:39,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 4 to mira0437250-387 2014-07-01 17:13:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 6 to mira0437250-387 2014-07-01 17:13:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: finishing write tid 7 to mira0437250-387 2014-07-01 17:13:39,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 387 v 1638 (ObjNum 1577 snap 406 seq_num 1577) dirty exists 2014-07-01 17:13:39,591.591 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:13:39,591.591 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:13:39,592.592 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:13:39,592.592 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:40,175.175 INFO:teuthology.task.rados.rados.0.mira043.stdout:3461: done (1 left) 2014-07-01 17:13:40,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:3463: done (0 left) 2014-07-01 17:13:40,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: write oid 101 current snap is 407 2014-07-01 17:13:40,176.176 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: seq_num 1579 ranges {675727=512433,1863157=447975,2908523=744953,3875726=1} 2014-07-01 17:13:40,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: writing mira0437250-101 from 675727 to 1188160 tid 1 2014-07-01 17:13:40,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: writing mira0437250-101 from 1863157 to 2311132 tid 2 2014-07-01 17:13:40,234.234 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:40,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: writing mira0437250-101 from 2908523 to 3653476 tid 3 2014-07-01 17:13:40,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: writing mira0437250-101 from 3875726 to 3875727 tid 4 2014-07-01 17:13:40,371.371 INFO:teuthology.task.rados.rados.0.mira043.stdout:3465: rollback oid 365 current snap is 407 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 365 to 404 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 1 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 2 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3466: read oid 237 snap 403 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: write oid 44 current snap is 407 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: seq_num 1580 ranges {469132=767886,1669131=1} 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0437250-44 from 469132 to 1237018 tid 1 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: writing mira0437250-44 from 1669131 to 1669132 tid 2 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: rollback oid 323 current snap is 407 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 323 to 375 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 3 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 4 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 6 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: finishing write tid 7 to mira0437250-101 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 101 v 1216 (ObjNum 1579 snap 407 seq_num 1579) dirty exists 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 365 v 1454 (ObjNum 850 snap 141 seq_num 850) dirty exists 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3464: done (4 left) 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3465: done (3 left) 2014-07-01 17:13:40,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3466: done (2 left) 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: write oid 421 current snap is 407 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: seq_num 1581 ranges {564362=491114,1364361=1} 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: writing mira0437250-421 from 564362 to 1055476 tid 1 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: writing mira0437250-421 from 1364361 to 1364362 tid 2 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3470: snap_remove snap 380 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 323 v 1261 (ObjNum 1440 snap 346 seq_num 1440) dirty exists 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 1 to mira0437250-44 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 2 to mira0437250-44 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 4 to mira0437250-44 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: finishing write tid 5 to mira0437250-44 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 44 v 1156 (ObjNum 1580 snap 407 seq_num 1580) dirty exists 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 1 to mira0437250-421 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 2 to mira0437250-421 2014-07-01 17:13:40,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 4 to mira0437250-421 2014-07-01 17:13:40,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: finishing write tid 5 to mira0437250-421 2014-07-01 17:13:40,382.382 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 1817 (ObjNum 1581 snap 407 seq_num 1581) dirty exists 2014-07-01 17:13:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 1 to mira0435601-351 2014-07-01 17:13:40,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 2 to mira0435601-351 2014-07-01 17:13:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 3 to mira0435601-351 2014-07-01 17:13:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 5 to mira0435601-351 2014-07-01 17:13:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 6 to mira0435601-351 2014-07-01 17:13:40,568.568 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 409 (ObjNum 2357 snap 0 seq_num 2357) dirty exists 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: finishing write tid 1 to mira0435601-421 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: finishing write tid 2 to mira0435601-421 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: finishing write tid 4 to mira0435601-421 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: finishing write tid 5 to mira0435601-421 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 421 v 360 (ObjNum 2358 snap 0 seq_num 2358) dirty exists 2014-07-01 17:13:40,569.569 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 1 to mira0435601-109 2014-07-01 17:13:40,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 2 to mira0435601-109 2014-07-01 17:13:40,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: done (15 left) 2014-07-01 17:13:40,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: done (14 left) 2014-07-01 17:13:40,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: done (13 left) 2014-07-01 17:13:40,570.570 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: done (12 left) 2014-07-01 17:13:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: done (11 left) 2014-07-01 17:13:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:3834: done (10 left) 2014-07-01 17:13:40,773.773 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: done (9 left) 2014-07-01 17:13:40,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: done (8 left) 2014-07-01 17:13:40,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3837: done (7 left) 2014-07-01 17:13:40,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: done (6 left) 2014-07-01 17:13:40,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: done (5 left) 2014-07-01 17:13:40,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3845: read oid 224 snap -1 2014-07-01 17:13:40,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 3 to mira0435601-109 2014-07-01 17:13:40,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 4 to mira0435601-109 2014-07-01 17:13:40,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 6 to mira0435601-109 2014-07-01 17:13:40,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: finishing write tid 7 to mira0435601-109 2014-07-01 17:13:40,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 243 (ObjNum 2359 snap 0 seq_num 2359) dirty exists 2014-07-01 17:13:40,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 1 to mira0435601-23 2014-07-01 17:13:40,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 2 to mira0435601-23 2014-07-01 17:13:40,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 4 to mira0435601-23 2014-07-01 17:13:40,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 5 to mira0435601-23 2014-07-01 17:13:40,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 233 (ObjNum 2360 snap 0 seq_num 2360) dirty exists 2014-07-01 17:13:40,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 1 to mira0435601-154 2014-07-01 17:13:40,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 2 to mira0435601-154 2014-07-01 17:13:40,777.777 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 3 to mira0435601-154 2014-07-01 17:13:40,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 4 to mira0435601-154 2014-07-01 17:13:40,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 6 to mira0435601-154 2014-07-01 17:13:40,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: finishing write tid 7 to mira0435601-154 2014-07-01 17:13:40,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 154 v 386 (ObjNum 2361 snap 0 seq_num 2361) dirty exists 2014-07-01 17:13:40,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3844: expect (ObjNum 1908 snap 0 seq_num 1908) 2014-07-01 17:13:41,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3467: done (3 left) 2014-07-01 17:13:41,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3468: done (2 left) 2014-07-01 17:13:41,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3469: done (1 left) 2014-07-01 17:13:41,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3470: done (0 left) 2014-07-01 17:13:41,216.216 INFO:teuthology.task.rados.rados.0.mira043.stdout:3471: delete oid 357 current snap is 407 2014-07-01 17:13:41,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:3471: done (0 left) 2014-07-01 17:13:41,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: rollback oid 69 current snap is 407 2014-07-01 17:13:41,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 69 to 375 2014-07-01 17:13:41,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: write oid 58 current snap is 407 2014-07-01 17:13:41,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: seq_num 1583 ranges {568326=435747,1427270=480179,2341776=669843,3368325=1} 2014-07-01 17:13:41,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: writing mira0437250-58 from 568326 to 1004073 tid 1 2014-07-01 17:13:41,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: writing mira0437250-58 from 1427270 to 1907449 tid 2 2014-07-01 17:13:41,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: writing mira0437250-58 from 2341776 to 3011619 tid 3 2014-07-01 17:13:41,429.429 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: writing mira0437250-58 from 3368325 to 3368326 tid 4 2014-07-01 17:13:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: write oid 376 current snap is 407 2014-07-01 17:13:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: seq_num 1584 ranges {664579=691199,1789977=701798,3064578=1} 2014-07-01 17:13:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: writing mira0437250-376 from 664579 to 1355778 tid 1 2014-07-01 17:13:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: writing mira0437250-376 from 1789977 to 2491775 tid 2 2014-07-01 17:13:41,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: writing mira0437250-376 from 3064578 to 3064579 tid 3 2014-07-01 17:13:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: read oid 151 snap -1 2014-07-01 17:13:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 69 v 0 (ObjNum 17027408 snap 0 seq_num 32) dirty dne 2014-07-01 17:13:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 1 to mira0437250-58 2014-07-01 17:13:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 2 to mira0437250-58 2014-07-01 17:13:41,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 3 to mira0437250-58 2014-07-01 17:13:41,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 4 to mira0437250-58 2014-07-01 17:13:41,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3472: done (3 left) 2014-07-01 17:13:41,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: snap_remove snap 397 2014-07-01 17:13:41,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 6 to mira0437250-58 2014-07-01 17:13:41,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: finishing write tid 7 to mira0437250-58 2014-07-01 17:13:41,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 1752 (ObjNum 1583 snap 407 seq_num 1583) dirty exists 2014-07-01 17:13:41,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: finishing write tid 1 to mira0437250-376 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: finishing write tid 2 to mira0437250-376 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: finishing write tid 3 to mira0437250-376 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: finishing write tid 5 to mira0437250-376 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: finishing write tid 6 to mira0437250-376 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 1740 (ObjNum 1584 snap 407 seq_num 1584) dirty exists 2014-07-01 17:13:41,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: expect (ObjNum 150 snap 0 seq_num 150) 2014-07-01 17:13:42,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3473: done (3 left) 2014-07-01 17:13:42,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3474: done (2 left) 2014-07-01 17:13:42,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3475: done (1 left) 2014-07-01 17:13:42,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3476: done (0 left) 2014-07-01 17:13:42,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: write oid 327 current snap is 407 2014-07-01 17:13:42,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: seq_num 1585 ranges {760833=414427,1655533=655642,2760832=1} 2014-07-01 17:13:42,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: writing mira0437250-327 from 760833 to 1175260 tid 1 2014-07-01 17:13:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: writing mira0437250-327 from 1655533 to 2311175 tid 2 2014-07-01 17:13:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: writing mira0437250-327 from 2760832 to 2760833 tid 3 2014-07-01 17:13:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: snap_create 2014-07-01 17:13:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: finishing write tid 1 to mira0437250-327 2014-07-01 17:13:42,407.407 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: finishing write tid 2 to mira0437250-327 2014-07-01 17:13:42,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: finishing write tid 3 to mira0437250-327 2014-07-01 17:13:42,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: finishing write tid 5 to mira0437250-327 2014-07-01 17:13:42,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: finishing write tid 6 to mira0437250-327 2014-07-01 17:13:42,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 923 (ObjNum 1585 snap 407 seq_num 1585) dirty exists 2014-07-01 17:13:42,408.408 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: expect (ObjNum 958 snap 0 seq_num 958) 2014-07-01 17:13:43,227.227 INFO:teuthology.task.rados.rados.0.mira043.stdout:3477: done (1 left) 2014-07-01 17:13:43,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3478: done (0 left) 2014-07-01 17:13:43,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3479: rollback oid 284 current snap is 408 2014-07-01 17:13:43,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 284 to 326 2014-07-01 17:13:43,228.228 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: snap_create 2014-07-01 17:13:43,252.252 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 1642 (ObjNum 781 snap 124 seq_num 781) dirty exists 2014-07-01 17:13:44,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3479: done (1 left) 2014-07-01 17:13:44,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3480: done (0 left) 2014-07-01 17:13:44,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:3481: rollback oid 176 current snap is 409 2014-07-01 17:13:44,240.240 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 176 to 375 2014-07-01 17:13:44,241.241 INFO:teuthology.task.rados.rados.0.mira043.stdout:3482: snap_remove snap 401 2014-07-01 17:13:44,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 176 v 1449 (ObjNum 1165 snap 245 seq_num 1165) dirty exists 2014-07-01 17:13:44,376.376 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 64' 2014-07-01 17:13:44,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: done (5 left) 2014-07-01 17:13:44,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: done (4 left) 2014-07-01 17:13:44,384.384 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: done (3 left) 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: done (2 left) 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3844: done (1 left) 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3845: done (0 left) 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3846: read oid 485 snap -1 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: read oid 381 snap -1 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3848: read oid 443 snap -1 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 36 current snap is 0 2014-07-01 17:13:44,406.406 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: seq_num 2362 ranges {4952064=552960,5505024=401408} 2014-07-01 17:13:44,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: writing mira0435601-36 from 4952064 to 5505024 tid 1 2014-07-01 17:13:44,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: writing mira0435601-36 from 5505024 to 5906432 tid 2 2014-07-01 17:13:44,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3850: read oid 184 snap -1 2014-07-01 17:13:44,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3846: done (4 left) 2014-07-01 17:13:44,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3851: delete oid 136 current snap is 0 2014-07-01 17:13:44,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3848: expect (ObjNum 2244 snap 0 seq_num 2244) 2014-07-01 17:13:45,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:3481: done (1 left) 2014-07-01 17:13:45,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:3482: done (0 left) 2014-07-01 17:13:45,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:3483: read oid 245 snap -1 2014-07-01 17:13:45,245.245 INFO:teuthology.task.rados.rados.0.mira043.stdout:3484: snap_create 2014-07-01 17:13:45,267.267 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 64 2014-07-01 17:13:46,248.248 INFO:teuthology.task.rados.rados.0.mira043.stdout:3483: done (1 left) 2014-07-01 17:13:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3484: done (0 left) 2014-07-01 17:13:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: read oid 495 snap 304 2014-07-01 17:13:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: rollback oid 473 current snap is 410 2014-07-01 17:13:46,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 473 to 407 2014-07-01 17:13:46,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: write oid 142 current snap is 410 2014-07-01 17:13:46,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: seq_num 1586 ranges {553214=1} 2014-07-01 17:13:46,251.251 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: writing mira0437250-142 from 553214 to 553215 tid 1 2014-07-01 17:13:46,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: read oid 342 snap -1 2014-07-01 17:13:46,412.412 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: read oid 185 snap -1 2014-07-01 17:13:46,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: snap_create 2014-07-01 17:13:46,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 0 (ObjNum 34495032 snap 0 seq_num 15965296) dirty dne 2014-07-01 17:13:46,413.413 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: expect (ObjNum 827 snap 132 seq_num 827) 2014-07-01 17:13:46,426.426 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 1 to mira0437250-142 2014-07-01 17:13:46,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 3 to mira0437250-142 2014-07-01 17:13:46,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: finishing write tid 4 to mira0437250-142 2014-07-01 17:13:46,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 142 v 1346 (ObjNum 1586 snap 410 seq_num 1586) dirty exists 2014-07-01 17:13:46,428.428 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: expect (ObjNum 1340 snap 309 seq_num 1340) 2014-07-01 17:13:47,472.472 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: expect (ObjNum 1857 snap 0 seq_num 1857) 2014-07-01 17:13:48,189.189 INFO:teuthology.task.rados.rados.0.mira043.stdout:3485: done (5 left) 2014-07-01 17:13:48,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3486: done (4 left) 2014-07-01 17:13:48,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3487: done (3 left) 2014-07-01 17:13:48,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3488: done (2 left) 2014-07-01 17:13:48,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3489: done (1 left) 2014-07-01 17:13:48,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:3490: done (0 left) 2014-07-01 17:13:48,191.191 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: write oid 494 current snap is 411 2014-07-01 17:13:48,192.192 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: seq_num 1587 ranges {765819=1} 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: writing mira0437250-494 from 765819 to 765820 tid 1 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3492: delete oid 348 current snap is 411 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 1 to mira0437250-494 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 3 to mira0437250-494 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: finishing write tid 4 to mira0437250-494 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 494 v 846 (ObjNum 1587 snap 411 seq_num 1587) dirty exists 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3491: done (1 left) 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3492: done (0 left) 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: snap_create 2014-07-01 17:13:48,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3493: done (0 left) 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3494: read oid 316 snap -1 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3495: rollback oid 172 current snap is 412 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 172 to 329 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3496: read oid 243 snap -1 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: delete oid 321 current snap is 412 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3494: done (3 left) 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3497: done (2 left) 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: rollback oid 471 current snap is 412 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 471 to 304 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: snap_remove snap 344 2014-07-01 17:13:48,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 1671 (ObjNum 1381 snap 313 seq_num 1381) dirty exists 2014-07-01 17:13:48,360.360 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 1053 (ObjNum 1302 snap 295 seq_num 1302) dirty exists 2014-07-01 17:13:49,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3495: done (3 left) 2014-07-01 17:13:49,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3496: done (2 left) 2014-07-01 17:13:49,277.277 INFO:teuthology.task.rados.rados.0.mira043.stdout:3498: done (1 left) 2014-07-01 17:13:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:3499: done (0 left) 2014-07-01 17:13:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: write oid 152 current snap is 412 2014-07-01 17:13:49,278.278 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: seq_num 1590 ranges {750710=461993,1852179=98531} 2014-07-01 17:13:49,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: writing mira0437250-152 from 750710 to 1212703 tid 1 2014-07-01 17:13:49,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: writing mira0437250-152 from 1852179 to 1950710 tid 2 2014-07-01 17:13:49,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3501: snap_remove snap 381 2014-07-01 17:13:49,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 1 to mira0437250-152 2014-07-01 17:13:49,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 2 to mira0437250-152 2014-07-01 17:13:49,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 4 to mira0437250-152 2014-07-01 17:13:49,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: finishing write tid 5 to mira0437250-152 2014-07-01 17:13:49,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 1455 (ObjNum 1590 snap 412 seq_num 1590) dirty exists 2014-07-01 17:13:49,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 1 to mira0435601-36 2014-07-01 17:13:49,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 2 to mira0435601-36 2014-07-01 17:13:49,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 4 to mira0435601-36 2014-07-01 17:13:49,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 5 to mira0435601-36 2014-07-01 17:13:49,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 36 v 394 (ObjNum 2362 snap 0 seq_num 2362) dirty exists 2014-07-01 17:13:49,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3850: expect (ObjNum 2196 snap 0 seq_num 2196) 2014-07-01 17:13:50,280.280 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:13:50,281.281 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:13:50,281.281 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_1 2014-07-01 17:13:50,282.282 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:13:50,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:3500: done (1 left) 2014-07-01 17:13:50,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3501: done (0 left) 2014-07-01 17:13:50,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: write oid 444 current snap is 412 2014-07-01 17:13:50,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: seq_num 1591 ranges {444915=585221,1644914=1} 2014-07-01 17:13:50,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: writing mira0437250-444 from 444915 to 1030136 tid 1 2014-07-01 17:13:50,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: writing mira0437250-444 from 1644914 to 1644915 tid 2 2014-07-01 17:13:50,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3503: delete oid 389 current snap is 412 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: finishing write tid 1 to mira0437250-444 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: finishing write tid 2 to mira0437250-444 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: finishing write tid 4 to mira0437250-444 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: finishing write tid 5 to mira0437250-444 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 980 (ObjNum 1591 snap 412 seq_num 1591) dirty exists 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3502: done (1 left) 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3503: done (0 left) 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: read oid 395 snap -1 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: write oid 192 current snap is 412 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: seq_num 1593 ranges {449903=564926,1508879=470451,2734830=619989,3649902=1} 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: writing mira0437250-192 from 449903 to 1014829 tid 1 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: writing mira0437250-192 from 1508879 to 1979330 tid 2 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: writing mira0437250-192 from 2734830 to 3354819 tid 3 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: writing mira0437250-192 from 3649902 to 3649903 tid 4 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3506: delete oid 449 current snap is 412 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3504: done (2 left) 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3506: done (1 left) 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: write oid 445 current snap is 412 2014-07-01 17:13:50,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: seq_num 1595 ranges {90985=1} 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: writing mira0437250-445 from 90985 to 90986 tid 1 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3508: snap_create 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: finishing write tid 1 to mira0437250-445 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: finishing write tid 3 to mira0437250-445 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: finishing write tid 4 to mira0437250-445 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 1541 (ObjNum 1595 snap 412 seq_num 1595) dirty exists 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 1 to mira0437250-192 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: done (4 left) 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3848: done (3 left) 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: done (2 left) 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3850: done (1 left) 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3851: done (0 left) 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: read oid 488 snap -1 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: read oid 29 snap -1 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3854: read oid 237 snap -1 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3855: read oid 109 snap -1 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 404 current snap is 0 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: seq_num 2364 ranges {532480=745472,1277952=577536,1855488=225280} 2014-07-01 17:13:50,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: writing mira0435601-404 from 532480 to 1277952 tid 1 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: writing mira0435601-404 from 1277952 to 1855488 tid 2 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: writing mira0435601-404 from 1855488 to 2080768 tid 3 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 342 current snap is 0 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: seq_num 2365 ranges {0=372736} 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: writing mira0435601-342 from 0 to 372736 tid 1 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: read oid 258 snap -1 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3859: read oid 168 snap -1 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 227 current snap is 0 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: seq_num 2366 ranges {4481024=487424,4968448=745472,5713920=57344} 2014-07-01 17:13:50,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: writing mira0435601-227 from 4481024 to 4968448 tid 1 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 2 to mira0437250-192 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 3 to mira0437250-192 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 4 to mira0437250-192 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 6 to mira0437250-192 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: finishing write tid 7 to mira0437250-192 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 1494 (ObjNum 1593 snap 412 seq_num 1593) dirty exists 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: writing mira0435601-227 from 4968448 to 5713920 tid 2 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: writing mira0435601-227 from 5713920 to 5771264 tid 3 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 298 current snap is 0 2014-07-01 17:13:50,484.484 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: seq_num 2367 ranges {0=565248,565248=401408} 2014-07-01 17:13:50,492.492 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: writing mira0435601-298 from 0 to 565248 tid 1 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: writing mira0435601-298 from 565248 to 966656 tid 2 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 239 current snap is 0 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: seq_num 2368 ranges {24576=438272,462848=401408} 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0435601-239 from 24576 to 462848 tid 1 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0435601-239 from 462848 to 864256 tid 2 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3863: read oid 421 snap -1 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3864: read oid 467 snap -1 2014-07-01 17:13:50,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: finishing write tid 1 to mira0435601-404 2014-07-01 17:13:50,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: expect (ObjNum 2094 snap 0 seq_num 2094) 2014-07-01 17:13:51,067.067 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:13:51,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:3505: done (2 left) 2014-07-01 17:13:51,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:3507: done (1 left) 2014-07-01 17:13:51,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:3508: done (0 left) 2014-07-01 17:13:51,332.332 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: write oid 237 current snap is 413 2014-07-01 17:13:51,333.333 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: seq_num 1596 ranges {551144=765011,1351143=1} 2014-07-01 17:13:51,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: writing mira0437250-237 from 551144 to 1316155 tid 1 2014-07-01 17:13:51,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: writing mira0437250-237 from 1351143 to 1351144 tid 2 2014-07-01 17:13:51,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: write oid 485 current snap is 413 2014-07-01 17:13:51,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: seq_num 1597 ranges {647398=400000} 2014-07-01 17:13:51,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: writing mira0437250-485 from 647398 to 1047398 tid 1 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: delete oid 473 current snap is 413 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3511: done (2 left) 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3512: rollback oid 198 current snap is 413 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 198 to 375 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3513: snap_create 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 1 to mira0437250-237 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 2 to mira0437250-237 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 4 to mira0437250-237 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 198 v 975 (ObjNum 938 snap 171 seq_num 938) dirty exists 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: finishing write tid 5 to mira0437250-237 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 1329 (ObjNum 1596 snap 413 seq_num 1596) dirty exists 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 1 to mira0437250-485 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 3 to mira0437250-485 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: finishing write tid 4 to mira0437250-485 2014-07-01 17:13:51,497.497 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 1471 (ObjNum 1597 snap 413 seq_num 1597) dirty exists 2014-07-01 17:13:52,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: finishing write tid 2 to mira0435601-404 2014-07-01 17:13:52,015.015 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: finishing write tid 3 to mira0435601-404 2014-07-01 17:13:52,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: finishing write tid 5 to mira0435601-404 2014-07-01 17:13:52,016.016 INFO:teuthology.task.rados.rados.0.mira043.stdout:3859: expect (ObjNum 2269 snap 0 seq_num 2269) 2014-07-01 17:13:52,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: finishing write tid 6 to mira0435601-404 2014-07-01 17:13:52,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 404 v 266 (ObjNum 2364 snap 0 seq_num 2364) dirty exists 2014-07-01 17:13:52,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: expect (ObjNum 1939 snap 0 seq_num 1939) 2014-07-01 17:13:52,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3509: done (3 left) 2014-07-01 17:13:52,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3510: done (2 left) 2014-07-01 17:13:52,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3512: done (1 left) 2014-07-01 17:13:52,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3513: done (0 left) 2014-07-01 17:13:52,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: read oid 135 snap -1 2014-07-01 17:13:52,347.347 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: read oid 114 snap 304 2014-07-01 17:13:52,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: write oid 20 current snap is 414 2014-07-01 17:13:52,348.348 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: seq_num 1599 ranges {535009=584296,1783559=678118,2535008=1} 2014-07-01 17:13:52,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: writing mira0437250-20 from 535009 to 1119305 tid 1 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: writing mira0437250-20 from 1783559 to 2461677 tid 2 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: writing mira0437250-20 from 2535008 to 2535009 tid 3 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: snap_remove snap 389 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: expect (ObjNum 1195 snap 254 seq_num 1195) 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: expect (ObjNum 1131 snap 230 seq_num 1131) 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 1 to mira0437250-20 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 2 to mira0437250-20 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 3 to mira0437250-20 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 5 to mira0437250-20 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: finishing write tid 6 to mira0437250-20 2014-07-01 17:13:52,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 1221 (ObjNum 1599 snap 414 seq_num 1599) dirty exists 2014-07-01 17:13:53,353.353 INFO:teuthology.task.rados.rados.0.mira043.stdout:3514: done (3 left) 2014-07-01 17:13:53,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3515: done (2 left) 2014-07-01 17:13:53,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3516: done (1 left) 2014-07-01 17:13:53,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3517: done (0 left) 2014-07-01 17:13:53,354.354 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: delete oid 320 current snap is 414 2014-07-01 17:13:53,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3518: done (0 left) 2014-07-01 17:13:53,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: read oid 309 snap 355 2014-07-01 17:13:53,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3520: read oid 347 snap -1 2014-07-01 17:13:53,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3521: snap_remove snap 366 2014-07-01 17:13:53,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: expect (ObjNum 807 snap 130 seq_num 807) 2014-07-01 17:13:54,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: expect (ObjNum 1042 snap 0 seq_num 1042) 2014-07-01 17:13:54,661.661 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_1 pgp_num 86' 2014-07-01 17:13:55,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:3520: expect (ObjNum 1221 snap 261 seq_num 1221) 2014-07-01 17:13:55,435.435 INFO:teuthology.orchestra.run.mira079.stderr:set pool 4 pgp_num to 86 2014-07-01 17:13:55,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3519: done (2 left) 2014-07-01 17:13:55,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3520: done (1 left) 2014-07-01 17:13:55,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3521: done (0 left) 2014-07-01 17:13:55,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3522: delete oid 288 current snap is 414 2014-07-01 17:13:55,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3522: done (0 left) 2014-07-01 17:13:55,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: write oid 97 current snap is 414 2014-07-01 17:13:55,499.499 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: seq_num 1602 ranges {787675=535805,2090884=488704,3187674=1} 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: writing mira0437250-97 from 787675 to 1323480 tid 1 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: writing mira0437250-97 from 2090884 to 2579588 tid 2 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: writing mira0437250-97 from 3187674 to 3187675 tid 3 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: write oid 469 current snap is 414 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: seq_num 1603 ranges {731480=659033,2051543=593971,3321162=610318} 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: writing mira0437250-469 from 731480 to 1390513 tid 1 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: writing mira0437250-469 from 2051543 to 2645514 tid 2 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: writing mira0437250-469 from 3321162 to 3931480 tid 3 2014-07-01 17:13:55,644.644 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: read oid 59 snap 304 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 1 to mira0437250-97 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 2 to mira0437250-97 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 3 to mira0437250-97 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3526: snap_remove snap 387 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 5 to mira0437250-97 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: finishing write tid 6 to mira0437250-97 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 1094 (ObjNum 1602 snap 414 seq_num 1602) dirty exists 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: finishing write tid 1 to mira0437250-469 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: finishing write tid 2 to mira0437250-469 2014-07-01 17:13:55,645.645 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: expect (ObjNum 1223 snap 263 seq_num 1223) 2014-07-01 17:13:56,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: finishing write tid 3 to mira0437250-469 2014-07-01 17:13:56,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: finishing write tid 5 to mira0437250-469 2014-07-01 17:13:56,590.590 INFO:teuthology.task.rados.rados.0.mira043.stdout:3523: done (3 left) 2014-07-01 17:13:56,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3525: done (2 left) 2014-07-01 17:13:56,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3526: done (1 left) 2014-07-01 17:13:56,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3527: read oid 78 snap -1 2014-07-01 17:13:56,591.591 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: finishing write tid 6 to mira0437250-469 2014-07-01 17:13:56,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 469 v 1692 (ObjNum 1603 snap 414 seq_num 1603) dirty exists 2014-07-01 17:13:56,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3524: done (1 left) 2014-07-01 17:13:56,592.592 INFO:teuthology.task.rados.rados.0.mira043.stdout:3528: delete oid 414 current snap is 414 2014-07-01 17:13:56,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:3527: done (1 left) 2014-07-01 17:13:56,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3528: done (0 left) 2014-07-01 17:13:56,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: write oid 232 current snap is 414 2014-07-01 17:13:56,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: seq_num 1605 ranges {488916=400000} 2014-07-01 17:13:56,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: writing mira0437250-232 from 488916 to 888916 tid 1 2014-07-01 17:13:56,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: snap_create 2014-07-01 17:13:56,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: finishing write tid 1 to mira0437250-232 2014-07-01 17:13:56,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: finishing write tid 3 to mira0437250-232 2014-07-01 17:13:56,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: finishing write tid 4 to mira0437250-232 2014-07-01 17:13:56,761.761 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 232 v 1523 (ObjNum 1605 snap 414 seq_num 1605) dirty exists 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: finishing write tid 1 to mira0435601-227 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: finishing write tid 2 to mira0435601-227 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: finishing write tid 3 to mira0435601-227 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: finishing write tid 5 to mira0435601-227 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: finishing write tid 6 to mira0435601-227 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 238 (ObjNum 2366 snap 0 seq_num 2366) dirty exists 2014-07-01 17:13:56,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3854: expect (ObjNum 2031 snap 0 seq_num 2031) 2014-07-01 17:13:57,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3529: done (1 left) 2014-07-01 17:13:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:3530: done (0 left) 2014-07-01 17:13:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:3531: rollback oid 320 current snap is 415 2014-07-01 17:13:57,448.448 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 320 to 408 2014-07-01 17:13:57,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: snap_remove snap 400 2014-07-01 17:13:57,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 320 v 1038 (ObjNum 319 snap 0 seq_num 319) dirty exists 2014-07-01 17:13:58,450.450 INFO:teuthology.task.rados.rados.0.mira043.stdout:3531: done (1 left) 2014-07-01 17:13:58,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3532: done (0 left) 2014-07-01 17:13:58,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: read oid 379 snap -1 2014-07-01 17:13:58,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: snap_remove snap 405 2014-07-01 17:13:58,493.493 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: expect (ObjNum 1576 snap 405 seq_num 1576) 2014-07-01 17:13:59,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3533: done (1 left) 2014-07-01 17:13:59,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3534: done (0 left) 2014-07-01 17:13:59,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3535: read oid 469 snap -1 2014-07-01 17:13:59,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: write oid 226 current snap is 415 2014-07-01 17:13:59,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: seq_num 1606 ranges {681297=459119,1904597=555981,2681296=1} 2014-07-01 17:13:59,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: writing mira0437250-226 from 681297 to 1140416 tid 1 2014-07-01 17:13:59,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3863: expect (ObjNum 2358 snap 0 seq_num 2358) 2014-07-01 17:13:59,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: writing mira0437250-226 from 1904597 to 2460578 tid 2 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: writing mira0437250-226 from 2681296 to 2681297 tid 3 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: read oid 261 snap 384 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: read oid 67 snap -1 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3539: rollback oid 64 current snap is 415 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 64 to 371 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: read oid 417 snap -1 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: write oid 45 current snap is 415 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: seq_num 1607 ranges {776527=583371,1886505=490022} 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: writing mira0437250-45 from 776527 to 1359898 tid 1 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: writing mira0437250-45 from 1886505 to 2376527 tid 2 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: read oid 83 snap -1 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 64 v 0 (ObjNum 0 snap 0 seq_num 0) dirty dne 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3539: done (7 left) 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: snap_remove snap 404 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: finishing write tid 1 to mira0437250-226 2014-07-01 17:13:59,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: finishing write tid 2 to mira0437250-226 2014-07-01 17:13:59,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: finishing write tid 3 to mira0437250-226 2014-07-01 17:13:59,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: finishing write tid 5 to mira0437250-226 2014-07-01 17:13:59,632.632 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: expect (ObjNum 1288 snap 292 seq_num 1288) 2014-07-01 17:13:59,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:3535: expect (ObjNum 1603 snap 414 seq_num 1603) 2014-07-01 17:14:00,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3855: expect (ObjNum 2359 snap 0 seq_num 2359) 2014-07-01 17:14:00,450.450 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:00,450.450 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:00,450.450 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.620613 2014-07-01 17:14:00,450.450 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 0.620612984718' 2014-07-01 17:14:01,513.513 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 0.620613 (8406722) 2014-07-01 17:14:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: finishing write tid 1 to mira0437250-45 2014-07-01 17:14:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: finishing write tid 6 to mira0437250-226 2014-07-01 17:14:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 226 v 1015 (ObjNum 1606 snap 415 seq_num 1606) dirty exists 2014-07-01 17:14:01,549.549 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: expect (ObjNum 260 snap 0 seq_num 260) 2014-07-01 17:14:01,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: finishing write tid 1 to mira0435601-342 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: finishing write tid 3 to mira0435601-342 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: finishing write tid 4 to mira0435601-342 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 242 (ObjNum 2365 snap 0 seq_num 2365) dirty exists 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: finishing write tid 1 to mira0435601-298 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: finishing write tid 2 to mira0435601-298 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: finishing write tid 4 to mira0435601-298 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: done (12 left) 2014-07-01 17:14:01,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: done (11 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3854: done (10 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3855: done (9 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: done (8 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: done (7 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: done (6 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3859: done (5 left) 2014-07-01 17:14:01,780.780 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: done (4 left) 2014-07-01 17:14:01,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3863: done (3 left) 2014-07-01 17:14:01,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: read oid 418 snap -1 2014-07-01 17:14:01,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: finishing write tid 5 to mira0435601-298 2014-07-01 17:14:01,781.781 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 165 (ObjNum 2367 snap 0 seq_num 2367) dirty exists 2014-07-01 17:14:01,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 1 to mira0435601-239 2014-07-01 17:14:01,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 2 to mira0435601-239 2014-07-01 17:14:01,946.946 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 4 to mira0435601-239 2014-07-01 17:14:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: done (3 left) 2014-07-01 17:14:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3866: read oid 83 snap -1 2014-07-01 17:14:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 5 to mira0435601-239 2014-07-01 17:14:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 168 (ObjNum 2368 snap 0 seq_num 2368) dirty exists 2014-07-01 17:14:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3864: expect (ObjNum 1374 snap 0 seq_num 1374) 2014-07-01 17:14:02,236.236 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: expect (ObjNum 1525 snap 381 seq_num 1525) 2014-07-01 17:14:02,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3866: expect (ObjNum 2202 snap 0 seq_num 2202) 2014-07-01 17:14:03,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: expect (ObjNum 1077 snap 216 seq_num 1077) 2014-07-01 17:14:03,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: finishing write tid 2 to mira0437250-45 2014-07-01 17:14:03,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: finishing write tid 4 to mira0437250-45 2014-07-01 17:14:03,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: finishing write tid 5 to mira0437250-45 2014-07-01 17:14:03,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 45 v 2191 (ObjNum 1607 snap 415 seq_num 1607) dirty exists 2014-07-01 17:14:03,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3535: done (7 left) 2014-07-01 17:14:03,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3536: done (6 left) 2014-07-01 17:14:03,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3537: done (5 left) 2014-07-01 17:14:03,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3538: done (4 left) 2014-07-01 17:14:03,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3540: done (3 left) 2014-07-01 17:14:03,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3541: done (2 left) 2014-07-01 17:14:03,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3542: done (1 left) 2014-07-01 17:14:03,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3543: done (0 left) 2014-07-01 17:14:03,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3544: snap_remove snap 398 2014-07-01 17:14:04,559.559 INFO:teuthology.task.rados.rados.0.mira043.stdout:3544: done (0 left) 2014-07-01 17:14:04,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: read oid 290 snap 371 2014-07-01 17:14:04,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: snap_remove snap 414 2014-07-01 17:14:04,582.582 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: expect (ObjNum 2338 snap 0 seq_num 2338) 2014-07-01 17:14:04,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: expect (ObjNum 289 snap 0 seq_num 289) 2014-07-01 17:14:05,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3545: done (1 left) 2014-07-01 17:14:05,566.566 INFO:teuthology.task.rados.rados.0.mira043.stdout:3546: done (0 left) 2014-07-01 17:14:05,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: write oid 269 current snap is 415 2014-07-01 17:14:05,567.567 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: seq_num 1608 ranges {471757=438823,1483261=447795,2071756=1} 2014-07-01 17:14:05,580.580 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: writing mira0437250-269 from 471757 to 910580 tid 1 2014-07-01 17:14:05,598.598 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: writing mira0437250-269 from 1483261 to 1931056 tid 2 2014-07-01 17:14:05,600.600 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: writing mira0437250-269 from 2071756 to 2071757 tid 3 2014-07-01 17:14:05,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3548: rollback oid 475 current snap is 415 2014-07-01 17:14:05,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 475 to 382 2014-07-01 17:14:05,738.738 INFO:teuthology.task.rados.rados.0.mira043.stdout:3549: read oid 111 snap -1 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3550: delete oid 43 current snap is 415 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 475 v 1296 (ObjNum 1318 snap 302 seq_num 1318) dirty exists 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 1 to mira0437250-269 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 2 to mira0437250-269 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 3 to mira0437250-269 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 5 to mira0437250-269 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: finishing write tid 6 to mira0437250-269 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 269 v 976 (ObjNum 1608 snap 415 seq_num 1608) dirty exists 2014-07-01 17:14:05,739.739 INFO:teuthology.task.rados.rados.0.mira043.stdout:3547: done (3 left) 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3548: done (2 left) 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3549: done (1 left) 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3550: done (0 left) 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: read oid 375 snap -1 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: read oid 401 snap -1 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: write oid 351 current snap is 415 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: seq_num 1610 ranges {77767=1} 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: writing mira0437250-351 from 77767 to 77768 tid 1 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3554: read oid 171 snap 377 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: write oid 417 current snap is 415 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: seq_num 1611 ranges {572998=658108,1905913=409805,2877719=518090,3772997=1} 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0437250-417 from 572998 to 1231106 tid 1 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0437250-417 from 1905913 to 2315718 tid 2 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0437250-417 from 2877719 to 3395809 tid 3 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: writing mira0437250-417 from 3772997 to 3772998 tid 4 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: delete oid 166 current snap is 415 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 1 to mira0437250-351 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 3 to mira0437250-351 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: finishing write tid 4 to mira0437250-351 2014-07-01 17:14:05,740.740 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 812 (ObjNum 1610 snap 415 seq_num 1610) dirty exists 2014-07-01 17:14:05,741.741 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: expect (ObjNum 400 snap 0 seq_num 400) 2014-07-01 17:14:06,525.525 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:06,526.526 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:06,526.526 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 17:14:06,527.527 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:14:06,528.528 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:14:06,529.529 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 17:14:07,118.118 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: expect (ObjNum 1117 snap 224 seq_num 1117) 2014-07-01 17:14:08,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 1 to mira0437250-417 2014-07-01 17:14:08,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 2 to mira0437250-417 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 3 to mira0437250-417 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 4 to mira0437250-417 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 6 to mira0437250-417 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: finishing write tid 7 to mira0437250-417 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 417 v 985 (ObjNum 1611 snap 415 seq_num 1611) dirty exists 2014-07-01 17:14:08,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3551: done (5 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3552: done (4 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3553: done (3 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3554: done (2 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3555: done (1 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3556: done (0 left) 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: read oid 344 snap -1 2014-07-01 17:14:08,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3558: snap_remove snap 377 2014-07-01 17:14:08,104.104 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: expect (ObjNum 1483 snap 363 seq_num 1483) 2014-07-01 17:14:08,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: done (3 left) 2014-07-01 17:14:08,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3864: done (2 left) 2014-07-01 17:14:08,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: done (1 left) 2014-07-01 17:14:08,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout:3866: done (0 left) 2014-07-01 17:14:08,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3867: read oid 346 snap -1 2014-07-01 17:14:08,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 449 current snap is 0 2014-07-01 17:14:08,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: seq_num 2369 ranges {2105344=462848,2568192=544768,3112960=724992,3837952=335872} 2014-07-01 17:14:08,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: writing mira0435601-449 from 2105344 to 2568192 tid 1 2014-07-01 17:14:08,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: writing mira0435601-449 from 2568192 to 3112960 tid 2 2014-07-01 17:14:08,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: writing mira0435601-449 from 3112960 to 3837952 tid 3 2014-07-01 17:14:08,563.563 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: writing mira0435601-449 from 3837952 to 4173824 tid 4 2014-07-01 17:14:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3869: read oid 306 snap -1 2014-07-01 17:14:08,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3867: expect (ObjNum 1621 snap 0 seq_num 1621) 2014-07-01 17:14:08,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:3557: done (1 left) 2014-07-01 17:14:08,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3558: done (0 left) 2014-07-01 17:14:08,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: delete oid 107 current snap is 415 2014-07-01 17:14:08,654.654 INFO:teuthology.task.rados.rados.0.mira043.stdout:3559: done (0 left) 2014-07-01 17:14:08,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: rollback oid 298 current snap is 415 2014-07-01 17:14:08,655.655 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 298 to 394 2014-07-01 17:14:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: read oid 400 snap 361 2014-07-01 17:14:08,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: snap_create 2014-07-01 17:14:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: expect (ObjNum 991 snap 188 seq_num 991) 2014-07-01 17:14:08,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:14:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:3560: done (2 left) 2014-07-01 17:14:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:3561: done (1 left) 2014-07-01 17:14:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:3562: done (0 left) 2014-07-01 17:14:09,628.628 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: write oid 419 current snap is 416 2014-07-01 17:14:09,629.629 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: seq_num 1614 ranges {674239=609617,1474238=1} 2014-07-01 17:14:09,647.647 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: writing mira0437250-419 from 674239 to 1283856 tid 1 2014-07-01 17:14:09,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: writing mira0437250-419 from 1474238 to 1474239 tid 2 2014-07-01 17:14:09,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: write oid 23 current snap is 416 2014-07-01 17:14:09,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: seq_num 1615 ranges {618045=732846,2139850=78195} 2014-07-01 17:14:09,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: writing mira0437250-23 from 618045 to 1350891 tid 1 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: writing mira0437250-23 from 2139850 to 2218045 tid 2 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: write oid 466 current snap is 416 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: seq_num 1616 ranges {561851=436874,1433758=794828,2892997=68854} 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0437250-466 from 561851 to 998725 tid 1 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0437250-466 from 1433758 to 2228586 tid 2 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: writing mira0437250-466 from 2892997 to 2961851 tid 3 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3566: snap_create 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: finishing write tid 1 to mira0437250-419 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: finishing write tid 2 to mira0437250-419 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: finishing write tid 4 to mira0437250-419 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: finishing write tid 5 to mira0437250-419 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 1244 (ObjNum 1614 snap 416 seq_num 1614) dirty exists 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: finishing write tid 1 to mira0437250-23 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: finishing write tid 2 to mira0437250-23 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: finishing write tid 4 to mira0437250-23 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: finishing write tid 5 to mira0437250-23 2014-07-01 17:14:09,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 23 v 956 (ObjNum 1615 snap 416 seq_num 1615) dirty exists 2014-07-01 17:14:10,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 1 to mira0435601-449 2014-07-01 17:14:10,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 2 to mira0435601-449 2014-07-01 17:14:10,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 3 to mira0435601-449 2014-07-01 17:14:10,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 4 to mira0435601-449 2014-07-01 17:14:10,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 6 to mira0435601-449 2014-07-01 17:14:10,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: finishing write tid 7 to mira0435601-449 2014-07-01 17:14:10,091.091 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 449 v 293 (ObjNum 2369 snap 0 seq_num 2369) dirty exists 2014-07-01 17:14:10,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3869: expect (ObjNum 2270 snap 0 seq_num 2270) 2014-07-01 17:14:10,649.649 INFO:teuthology.task.rados.rados.0.mira043.stdout:3563: done (3 left) 2014-07-01 17:14:10,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3564: done (2 left) 2014-07-01 17:14:10,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3566: done (1 left) 2014-07-01 17:14:10,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3567: snap_create 2014-07-01 17:14:10,899.899 INFO:teuthology.task.rados.rados.0.mira043.stdout:3867: done (2 left) 2014-07-01 17:14:10,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: done (1 left) 2014-07-01 17:14:10,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3869: done (0 left) 2014-07-01 17:14:10,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: read oid 407 snap -1 2014-07-01 17:14:10,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 109 current snap is 0 2014-07-01 17:14:10,902.902 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: seq_num 2370 ranges {2387968=737280,3125248=724992,3850240=479232} 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: writing mira0435601-109 from 2387968 to 3125248 tid 1 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: writing mira0435601-109 from 3125248 to 3850240 tid 2 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: writing mira0435601-109 from 3850240 to 4329472 tid 3 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 403 current snap is 0 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: seq_num 2371 ranges {2203648=610304,2813952=708608,3522560=495616} 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: writing mira0435601-403 from 2203648 to 2813952 tid 1 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: writing mira0435601-403 from 2813952 to 3522560 tid 2 2014-07-01 17:14:11,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: writing mira0435601-403 from 3522560 to 4018176 tid 3 2014-07-01 17:14:11,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 409 current snap is 0 2014-07-01 17:14:11,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: seq_num 2372 ranges {10031104=684032,10715136=536576,11251712=266240} 2014-07-01 17:14:11,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: writing mira0435601-409 from 10031104 to 10715136 tid 1 2014-07-01 17:14:11,134.134 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: writing mira0435601-409 from 10715136 to 11251712 tid 2 2014-07-01 17:14:11,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: writing mira0435601-409 from 11251712 to 11517952 tid 3 2014-07-01 17:14:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: read oid 315 snap -1 2014-07-01 17:14:11,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: expect (ObjNum 2062 snap 0 seq_num 2062) 2014-07-01 17:14:11,471.471 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 1 to mira0437250-466 2014-07-01 17:14:11,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 2 to mira0437250-466 2014-07-01 17:14:11,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 3 to mira0437250-466 2014-07-01 17:14:11,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 5 to mira0437250-466 2014-07-01 17:14:11,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: finishing write tid 6 to mira0437250-466 2014-07-01 17:14:11,486.486 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 1136 (ObjNum 1616 snap 416 seq_num 1616) dirty exists 2014-07-01 17:14:11,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:3565: done (1 left) 2014-07-01 17:14:11,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:3567: done (0 left) 2014-07-01 17:14:11,661.661 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: delete oid 279 current snap is 418 2014-07-01 17:14:11,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3568: done (0 left) 2014-07-01 17:14:11,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: write oid 239 current snap is 418 2014-07-01 17:14:11,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: seq_num 1618 ranges {602934=532931,1627974=438067,2765272=637662} 2014-07-01 17:14:11,693.693 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: writing mira0437250-239 from 602934 to 1135865 tid 1 2014-07-01 17:14:11,710.710 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: writing mira0437250-239 from 1627974 to 2066041 tid 2 2014-07-01 17:14:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: writing mira0437250-239 from 2765272 to 3402934 tid 3 2014-07-01 17:14:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: snap_create 2014-07-01 17:14:11,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: finishing write tid 1 to mira0437250-239 2014-07-01 17:14:11,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: finishing write tid 2 to mira0437250-239 2014-07-01 17:14:11,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: finishing write tid 3 to mira0437250-239 2014-07-01 17:14:11,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: finishing write tid 5 to mira0437250-239 2014-07-01 17:14:11,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: finishing write tid 6 to mira0437250-239 2014-07-01 17:14:11,826.826 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 239 v 1585 (ObjNum 1618 snap 418 seq_num 1618) dirty exists 2014-07-01 17:14:11,994.994 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:11,994.994 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:11,994.994 INFO:teuthology.task.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.808066 2014-07-01 17:14:11,994.994 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 5 0.808065685133' 2014-07-01 17:14:12,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: finishing write tid 1 to mira0435601-109 2014-07-01 17:14:12,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: finishing write tid 2 to mira0435601-109 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: finishing write tid 3 to mira0435601-109 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: finishing write tid 5 to mira0435601-109 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: finishing write tid 6 to mira0435601-109 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 109 v 247 (ObjNum 2370 snap 0 seq_num 2370) dirty exists 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: finishing write tid 1 to mira0435601-403 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: done (4 left) 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: done (3 left) 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 171 current snap is 0 2014-07-01 17:14:12,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: seq_num 2373 ranges {4521984=557056,5079040=770048,5849088=32768} 2014-07-01 17:14:12,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: writing mira0435601-171 from 4521984 to 5079040 tid 1 2014-07-01 17:14:12,223.223 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: writing mira0435601-171 from 5079040 to 5849088 tid 2 2014-07-01 17:14:12,225.225 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: writing mira0435601-171 from 5849088 to 5881856 tid 3 2014-07-01 17:14:12,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 186 current snap is 0 2014-07-01 17:14:12,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: seq_num 2374 ranges {4067328=430080,4497408=548864,5046272=253952} 2014-07-01 17:14:12,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: writing mira0435601-186 from 4067328 to 4497408 tid 1 2014-07-01 17:14:12,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: writing mira0435601-186 from 4497408 to 5046272 tid 2 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: writing mira0435601-186 from 5046272 to 5300224 tid 3 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3877: read oid 121 snap -1 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: finishing write tid 2 to mira0435601-403 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: finishing write tid 3 to mira0435601-403 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: finishing write tid 5 to mira0435601-403 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: finishing write tid 6 to mira0435601-403 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 398 (ObjNum 2371 snap 0 seq_num 2371) dirty exists 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: finishing write tid 1 to mira0435601-409 2014-07-01 17:14:12,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: finishing write tid 2 to mira0435601-409 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: finishing write tid 3 to mira0435601-409 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: finishing write tid 5 to mira0435601-409 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: finishing write tid 6 to mira0435601-409 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 409 v 387 (ObjNum 2372 snap 0 seq_num 2372) dirty exists 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: finishing write tid 1 to mira0435601-171 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: finishing write tid 2 to mira0435601-171 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: finishing write tid 3 to mira0435601-171 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: finishing write tid 5 to mira0435601-171 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: finishing write tid 6 to mira0435601-171 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 171 v 354 (ObjNum 2373 snap 0 seq_num 2373) dirty exists 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: finishing write tid 1 to mira0435601-186 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: done (5 left) 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: done (4 left) 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: done (3 left) 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: done (2 left) 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: read oid 381 snap -1 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 192 current snap is 0 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: seq_num 2375 ranges {2002944=53248} 2014-07-01 17:14:12,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: writing mira0435601-192 from 2002944 to 2056192 tid 1 2014-07-01 17:14:12,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: read oid 226 snap -1 2014-07-01 17:14:12,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: finishing write tid 2 to mira0435601-186 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: finishing write tid 3 to mira0435601-186 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: finishing write tid 5 to mira0435601-186 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: finishing write tid 6 to mira0435601-186 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 186 v 223 (ObjNum 2374 snap 0 seq_num 2374) dirty exists 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: done (4 left) 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: read oid 356 snap -1 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: read oid 442 snap -1 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:3883: read oid 406 snap -1 2014-07-01 17:14:12,372.372 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 301 current snap is 0 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: seq_num 2376 ranges {4845568=778240} 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: writing mira0435601-301 from 4845568 to 5623808 tid 1 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3885: read oid 132 snap -1 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: finishing write tid 1 to mira0435601-192 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: finishing write tid 3 to mira0435601-192 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 361 current snap is 0 2014-07-01 17:14:12,373.373 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: seq_num 2377 ranges {1073152=647168} 2014-07-01 17:14:12,392.392 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: writing mira0435601-361 from 1073152 to 1720320 tid 1 2014-07-01 17:14:12,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: read oid 50 snap -1 2014-07-01 17:14:12,485.485 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: expect (ObjNum 1857 snap 0 seq_num 1857) 2014-07-01 17:14:12,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:3569: done (1 left) 2014-07-01 17:14:12,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:3570: done (0 left) 2014-07-01 17:14:12,666.666 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: snap_remove snap 406 2014-07-01 17:14:12,705.705 INFO:teuthology.orchestra.run.mira079.stderr:set osd.5 primary-affinity to 0.808066 (8529572) 2014-07-01 17:14:13,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:3571: done (0 left) 2014-07-01 17:14:13,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:3572: read oid 141 snap -1 2014-07-01 17:14:13,676.676 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: write oid 360 current snap is 419 2014-07-01 17:14:13,677.677 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: seq_num 1619 ranges {663091=1} 2014-07-01 17:14:13,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: writing mira0437250-360 from 663091 to 663092 tid 1 2014-07-01 17:14:13,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: read oid 329 snap -1 2014-07-01 17:14:13,683.683 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: snap_remove snap 379 2014-07-01 17:14:13,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: expect (ObjNum 328 snap 0 seq_num 328) 2014-07-01 17:14:14,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3883: expect (ObjNum 1402 snap 0 seq_num 1402) 2014-07-01 17:14:15,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: expect (ObjNum 953 snap 0 seq_num 953) 2014-07-01 17:14:16,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3572: expect (ObjNum 1264 snap 276 seq_num 1264) 2014-07-01 17:14:17,713.713 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:17,714.714 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:17,714.714 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool unique_pool_0 2014-07-01 17:14:17,715.715 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 1 to mira0437250-360 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 3 to mira0437250-360 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: finishing write tid 4 to mira0437250-360 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 360 v 1099 (ObjNum 1619 snap 419 seq_num 1619) dirty exists 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3572: done (3 left) 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3573: done (2 left) 2014-07-01 17:14:18,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3574: done (1 left) 2014-07-01 17:14:18,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3575: done (0 left) 2014-07-01 17:14:18,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: write oid 204 current snap is 419 2014-07-01 17:14:18,162.162 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: seq_num 1620 ranges {359345=1} 2014-07-01 17:14:18,163.163 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: writing mira0437250-204 from 359345 to 359346 tid 1 2014-07-01 17:14:18,319.319 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:14:18,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: snap_remove snap 409 2014-07-01 17:14:18,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: finishing write tid 1 to mira0437250-204 2014-07-01 17:14:18,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: finishing write tid 3 to mira0437250-204 2014-07-01 17:14:18,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: finishing write tid 4 to mira0437250-204 2014-07-01 17:14:18,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 204 v 1519 (ObjNum 1620 snap 419 seq_num 1620) dirty exists 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3576: done (1 left) 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3577: done (0 left) 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: write oid 429 current snap is 419 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: seq_num 1621 ranges {703152=400000} 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: writing mira0437250-429 from 703152 to 1103152 tid 1 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: snap_create 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: finishing write tid 1 to mira0437250-429 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: finishing write tid 3 to mira0437250-429 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: finishing write tid 4 to mira0437250-429 2014-07-01 17:14:18,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 1385 (ObjNum 1621 snap 419 seq_num 1621) dirty exists 2014-07-01 17:14:18,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: finishing write tid 4 to mira0435601-192 2014-07-01 17:14:18,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 192 v 356 (ObjNum 2375 snap 0 seq_num 2375) dirty exists 2014-07-01 17:14:18,999.999 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: expect (ObjNum 1763 snap 0 seq_num 1763) 2014-07-01 17:14:19,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3578: done (1 left) 2014-07-01 17:14:19,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3579: done (0 left) 2014-07-01 17:14:19,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: delete oid 281 current snap is 420 2014-07-01 17:14:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3580: done (0 left) 2014-07-01 17:14:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:3581: rollback oid 145 current snap is 420 2014-07-01 17:14:19,249.249 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 145 to 411 2014-07-01 17:14:19,250.250 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: snap_create 2014-07-01 17:14:19,273.273 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 145 v 1060 (ObjNum 1552 snap 398 seq_num 1552) dirty exists 2014-07-01 17:14:20,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3581: done (1 left) 2014-07-01 17:14:20,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3582: done (0 left) 2014-07-01 17:14:20,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: write oid 485 current snap is 421 2014-07-01 17:14:20,254.254 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: seq_num 1623 ranges {459563=730896,1593195=424115,2059562=1} 2014-07-01 17:14:20,276.276 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0437250-485 from 459563 to 1190459 tid 1 2014-07-01 17:14:20,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0437250-485 from 1593195 to 2017310 tid 2 2014-07-01 17:14:20,293.293 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: writing mira0437250-485 from 2059562 to 2059563 tid 3 2014-07-01 17:14:20,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3584: rollback oid 175 current snap is 421 2014-07-01 17:14:20,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 175 to 415 2014-07-01 17:14:20,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 1 to mira0437250-485 2014-07-01 17:14:20,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: delete oid 280 current snap is 421 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3585: done (2 left) 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3586: read oid 135 snap -1 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: read oid 83 snap 371 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: delete oid 174 current snap is 421 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 175 v 1083 (ObjNum 174 snap 0 seq_num 174) dirty exists 2014-07-01 17:14:20,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 2 to mira0437250-485 2014-07-01 17:14:20,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 3 to mira0437250-485 2014-07-01 17:14:20,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 5 to mira0437250-485 2014-07-01 17:14:20,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: finishing write tid 6 to mira0437250-485 2014-07-01 17:14:20,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 485 v 1518 (ObjNum 1623 snap 421 seq_num 1623) dirty exists 2014-07-01 17:14:20,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: expect (ObjNum 1077 snap 216 seq_num 1077) 2014-07-01 17:14:21,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3586: expect (ObjNum 1195 snap 254 seq_num 1195) 2014-07-01 17:14:22,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3583: done (4 left) 2014-07-01 17:14:22,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3584: done (3 left) 2014-07-01 17:14:22,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3586: done (2 left) 2014-07-01 17:14:22,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3587: done (1 left) 2014-07-01 17:14:22,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3588: done (0 left) 2014-07-01 17:14:22,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: snap_remove snap 368 2014-07-01 17:14:22,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3589: done (0 left) 2014-07-01 17:14:22,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: write oid 331 current snap is 421 2014-07-01 17:14:22,295.295 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: seq_num 1626 ranges {443428=683429,1632743=786124,3004451=238977} 2014-07-01 17:14:22,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: writing mira0437250-331 from 443428 to 1126857 tid 1 2014-07-01 17:14:22,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: writing mira0437250-331 from 1632743 to 2418867 tid 2 2014-07-01 17:14:22,356.356 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: writing mira0437250-331 from 3004451 to 3243428 tid 3 2014-07-01 17:14:22,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3591: read oid 495 snap -1 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: write oid 212 current snap is 421 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: seq_num 1627 ranges {788258=655234,1959356=456320,3018158=662834,3988257=1} 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: writing mira0437250-212 from 788258 to 1443492 tid 1 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: writing mira0437250-212 from 1959356 to 2415676 tid 2 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: writing mira0437250-212 from 3018158 to 3680992 tid 3 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: writing mira0437250-212 from 3988257 to 3988258 tid 4 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: write oid 437 current snap is 421 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: seq_num 1628 ranges {200863=1} 2014-07-01 17:14:22,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: writing mira0437250-437 from 200863 to 200864 tid 1 2014-07-01 17:14:22,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: write oid 333 current snap is 421 2014-07-01 17:14:22,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: seq_num 1629 ranges {544669=400000} 2014-07-01 17:14:22,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: writing mira0437250-333 from 544669 to 944669 tid 1 2014-07-01 17:14:22,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: write oid 50 current snap is 421 2014-07-01 17:14:22,480.480 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: seq_num 1630 ranges {489499=606743,1689498=1} 2014-07-01 17:14:22,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: writing mira0437250-50 from 489499 to 1096242 tid 1 2014-07-01 17:14:22,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: writing mira0437250-50 from 1689498 to 1689499 tid 2 2014-07-01 17:14:22,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: delete oid 367 current snap is 421 2014-07-01 17:14:22,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3591: expect (ObjNum 827 snap 132 seq_num 827) 2014-07-01 17:14:22,647.647 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pgp_num 76' 2014-07-01 17:14:22,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: finishing write tid 1 to mira0437250-331 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: finishing write tid 2 to mira0437250-331 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: finishing write tid 3 to mira0437250-331 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: finishing write tid 5 to mira0437250-331 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: finishing write tid 6 to mira0437250-331 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 1092 (ObjNum 1626 snap 421 seq_num 1626) dirty exists 2014-07-01 17:14:22,723.723 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 1 to mira0437250-212 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 2 to mira0437250-212 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: finishing write tid 1 to mira0437250-437 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: finishing write tid 3 to mira0437250-437 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 3 to mira0437250-212 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 4 to mira0437250-212 2014-07-01 17:14:22,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 6 to mira0437250-212 2014-07-01 17:14:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: finishing write tid 4 to mira0437250-437 2014-07-01 17:14:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 1737 (ObjNum 1628 snap 421 seq_num 1628) dirty exists 2014-07-01 17:14:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: finishing write tid 7 to mira0437250-212 2014-07-01 17:14:22,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 1150 (ObjNum 1627 snap 421 seq_num 1627) dirty exists 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 1 to mira0437250-333 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 3 to mira0437250-333 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: finishing write tid 4 to mira0437250-333 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 333 v 975 (ObjNum 1629 snap 421 seq_num 1629) dirty exists 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 1 to mira0437250-50 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 2 to mira0437250-50 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 4 to mira0437250-50 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: finishing write tid 5 to mira0437250-50 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 1804 (ObjNum 1630 snap 421 seq_num 1630) dirty exists 2014-07-01 17:14:22,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3590: done (6 left) 2014-07-01 17:14:22,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3591: done (5 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3592: done (4 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3593: done (3 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3594: done (2 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3595: done (1 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3596: done (0 left) 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3597: rollback oid 290 current snap is 421 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 290 to 304 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: delete oid 74 current snap is 421 2014-07-01 17:14:22,893.893 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 290 v 1393 (ObjNum 289 snap 0 seq_num 289) dirty exists 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3597: done (1 left) 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3598: done (0 left) 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: read oid 490 snap 359 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3600: rollback oid 366 current snap is 421 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 366 to 371 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: write oid 61 current snap is 421 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: seq_num 1633 ranges {589716=709676,2021556=662963,3254885=134831} 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: writing mira0437250-61 from 589716 to 1299392 tid 1 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: writing mira0437250-61 from 2021556 to 2684519 tid 2 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: writing mira0437250-61 from 3254885 to 3389716 tid 3 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: rollback oid 142 current snap is 421 2014-07-01 17:14:22,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 142 to 361 2014-07-01 17:14:22,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: expect (ObjNum 1255 snap 274 seq_num 1255) 2014-07-01 17:14:23,322.322 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pgp_num to 76 2014-07-01 17:14:23,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: expect (ObjNum 2308 snap 0 seq_num 2308) 2014-07-01 17:14:24,024.024 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 1 to mira0437250-61 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 366 v 1201 (ObjNum 1474 snap 360 seq_num 1474) dirty exists 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 2 to mira0437250-61 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 3 to mira0437250-61 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 5 to mira0437250-61 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 142 v 1395 (ObjNum 141 snap 0 seq_num 141) dirty exists 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: finishing write tid 6 to mira0437250-61 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 1282 (ObjNum 1633 snap 421 seq_num 1633) dirty exists 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3599: done (3 left) 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3600: done (2 left) 2014-07-01 17:14:24,025.025 INFO:teuthology.task.rados.rados.0.mira043.stdout:3601: done (1 left) 2014-07-01 17:14:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3602: done (0 left) 2014-07-01 17:14:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: write oid 319 current snap is 421 2014-07-01 17:14:24,026.026 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: seq_num 1634 ranges {684946=681480,2098568=484582,3084945=1} 2014-07-01 17:14:24,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: writing mira0437250-319 from 684946 to 1366426 tid 1 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: writing mira0437250-319 from 2098568 to 2583150 tid 2 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: writing mira0437250-319 from 3084945 to 3084946 tid 3 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: snap_remove snap 408 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 1 to mira0437250-319 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 2 to mira0437250-319 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 3 to mira0437250-319 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 5 to mira0437250-319 2014-07-01 17:14:24,210.210 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: finishing write tid 6 to mira0437250-319 2014-07-01 17:14:24,211.211 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 319 v 1865 (ObjNum 1634 snap 421 seq_num 1634) dirty exists 2014-07-01 17:14:24,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:3603: done (1 left) 2014-07-01 17:14:24,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:3604: done (0 left) 2014-07-01 17:14:24,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:3605: rollback oid 466 current snap is 421 2014-07-01 17:14:24,314.314 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 466 to 384 2014-07-01 17:14:24,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3606: rollback oid 384 current snap is 421 2014-07-01 17:14:24,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 384 to 413 2014-07-01 17:14:24,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3607: read oid 279 snap 392 2014-07-01 17:14:24,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: read oid 469 snap -1 2014-07-01 17:14:24,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: delete oid 13 current snap is 421 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 466 v 1139 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3605: done (4 left) 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3607: done (3 left) 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3609: done (2 left) 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: read oid 139 snap -1 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: read oid 230 snap -1 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3612: read oid 341 snap -1 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: read oid 229 snap -1 2014-07-01 17:14:24,393.393 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: delete oid 125 current snap is 421 2014-07-01 17:14:24,498.498 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: expect (ObjNum 1603 snap 414 seq_num 1603) 2014-07-01 17:14:27,173.173 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 384 v 2234 (ObjNum 1372 snap 311 seq_num 1372) dirty exists 2014-07-01 17:14:27,174.174 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: expect (ObjNum 692 snap 88 seq_num 692) 2014-07-01 17:14:28,337.337 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 1, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:28,338.338 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:28,338.338 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [4, 0, 2, 1, 3, 5] 2014-07-01 17:14:28,339.339 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 1' 2014-07-01 17:14:29,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: expect (ObjNum 1541 snap 390 seq_num 1541) 2014-07-01 17:14:29,575.575 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.1. 2014-07-01 17:14:29,688.688 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: expect (ObjNum 1082 snap 217 seq_num 1082) 2014-07-01 17:14:30,746.746 INFO:teuthology.task.rados.rados.0.mira043.stdout:3606: done (6 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3608: done (5 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3610: done (4 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3611: done (3 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3612: done (2 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3613: done (1 left) 2014-07-01 17:14:30,747.747 INFO:teuthology.task.rados.rados.0.mira043.stdout:3614: done (0 left) 2014-07-01 17:14:30,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: rollback oid 95 current snap is 421 2014-07-01 17:14:30,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 95 to 326 2014-07-01 17:14:30,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:3616: rollback oid 397 current snap is 421 2014-07-01 17:14:30,748.748 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 397 to 416 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3617: read oid 132 snap -1 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: write oid 292 current snap is 421 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: seq_num 1637 ranges {786186=1} 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: writing mira0437250-292 from 786186 to 786187 tid 1 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: read oid 495 snap 361 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: read oid 489 snap 407 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3621: snap_create 2014-07-01 17:14:30,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: expect (ObjNum 827 snap 132 seq_num 827) 2014-07-01 17:14:31,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: expect (ObjNum 803 snap 129 seq_num 803) 2014-07-01 17:14:31,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 632 (ObjNum 396 snap 0 seq_num 396) dirty exists 2014-07-01 17:14:31,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 95 v 999 (ObjNum 1396 snap 319 seq_num 1396) dirty exists 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 1 to mira0437250-292 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 3 to mira0437250-292 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: finishing write tid 4 to mira0437250-292 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 1809 (ObjNum 1637 snap 421 seq_num 1637) dirty exists 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3615: done (6 left) 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3616: done (5 left) 2014-07-01 17:14:31,803.803 INFO:teuthology.task.rados.rados.0.mira043.stdout:3617: done (4 left) 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3618: done (3 left) 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3619: done (2 left) 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3620: done (1 left) 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3621: done (0 left) 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: read oid 486 snap 421 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: rollback oid 56 current snap is 422 2014-07-01 17:14:31,804.804 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 56 to 393 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: write oid 285 current snap is 422 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: seq_num 1638 ranges {482440=1} 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: writing mira0437250-285 from 482440 to 482441 tid 1 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3625: snap_create 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:14:31,967.967 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: expect (ObjNum 1188 snap 251 seq_num 1188) 2014-07-01 17:14:33,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:3877: expect (ObjNum 893 snap 0 seq_num 893) 2014-07-01 17:14:33,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3885: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 1 to mira0435601-361 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: finishing write tid 1 to mira0435601-301 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: finishing write tid 3 to mira0435601-301 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 3 to mira0435601-361 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: finishing write tid 4 to mira0435601-301 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 301 v 327 (ObjNum 2376 snap 0 seq_num 2376) dirty exists 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 4 to mira0435601-361 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 361 v 351 (ObjNum 2377 snap 0 seq_num 2377) dirty exists 2014-07-01 17:14:33,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3877: done (10 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: done (9 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: done (8 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: done (7 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: done (6 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: done (5 left) 2014-07-01 17:14:33,698.698 INFO:teuthology.task.rados.rados.0.mira043.stdout:3883: done (4 left) 2014-07-01 17:14:33,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: done (3 left) 2014-07-01 17:14:33,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3885: done (2 left) 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: done (1 left) 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: done (0 left) 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 237 current snap is 0 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: seq_num 2378 ranges {3883008=675840,4558848=528384,5087232=495616,5582848=180224} 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: writing mira0435601-237 from 3883008 to 4558848 tid 1 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: writing mira0435601-237 from 4558848 to 5087232 tid 2 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: writing mira0435601-237 from 5087232 to 5582848 tid 3 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: writing mira0435601-237 from 5582848 to 5763072 tid 4 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 415 current snap is 0 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: seq_num 2379 ranges {2347008=704512} 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: writing mira0435601-415 from 2347008 to 3051520 tid 1 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 396 current snap is 0 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: seq_num 2380 ranges {4071424=417792,4489216=745472,5234688=458752} 2014-07-01 17:14:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: writing mira0435601-396 from 4071424 to 4489216 tid 1 2014-07-01 17:14:33,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: writing mira0435601-396 from 4489216 to 5234688 tid 2 2014-07-01 17:14:33,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: writing mira0435601-396 from 5234688 to 5693440 tid 3 2014-07-01 17:14:33,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:3891: read oid 50 snap -1 2014-07-01 17:14:33,887.887 INFO:teuthology.task.rados.rados.0.mira043.stdout:3892: read oid 424 snap -1 2014-07-01 17:14:33,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:3893: delete oid 306 current snap is 0 2014-07-01 17:14:34,583.583 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5] out_osds: [1] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:34,584.584 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:34,584.584 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1 2014-07-01 17:14:34,584.584 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 1' 2014-07-01 17:14:35,751.751 INFO:teuthology.task.rados.rados.0.mira043.stdout:3622: done (3 left) 2014-07-01 17:14:35,752.752 INFO:teuthology.task.rados.rados.0.mira043.stdout:3623: done (2 left) 2014-07-01 17:14:35,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3625: done (1 left) 2014-07-01 17:14:35,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3626: snap_create 2014-07-01 17:14:35,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3892: done (5 left) 2014-07-01 17:14:35,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3893: done (4 left) 2014-07-01 17:14:35,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 338 current snap is 0 2014-07-01 17:14:35,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: seq_num 2382 ranges {0=122880} 2014-07-01 17:14:35,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: writing mira0435601-338 from 0 to 122880 tid 1 2014-07-01 17:14:35,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 17:14:35,914.914 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: seq_num 2383 ranges {5779456=638976,6418432=401408} 2014-07-01 17:14:35,963.963 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: writing mira0435601-292 from 5779456 to 6418432 tid 1 2014-07-01 17:14:36,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: writing mira0435601-292 from 6418432 to 6819840 tid 2 2014-07-01 17:14:36,067.067 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: read oid 368 snap -1 2014-07-01 17:14:36,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: finishing write tid 1 to mira0435601-338 2014-07-01 17:14:36,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: finishing write tid 3 to mira0435601-338 2014-07-01 17:14:36,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: finishing write tid 4 to mira0435601-338 2014-07-01 17:14:36,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 338 v 302 (ObjNum 2382 snap 0 seq_num 2382) dirty exists 2014-07-01 17:14:36,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: done (6 left) 2014-07-01 17:14:36,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 17:14:36,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: seq_num 2384 ranges {7712768=667648,8380416=569344,8949760=684032,9633792=352256} 2014-07-01 17:14:36,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: writing mira0435601-11 from 7712768 to 8380416 tid 1 2014-07-01 17:14:36,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: writing mira0435601-11 from 8380416 to 8949760 tid 2 2014-07-01 17:14:36,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 1 to mira0437250-285 2014-07-01 17:14:36,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 3 to mira0437250-285 2014-07-01 17:14:36,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: finishing write tid 4 to mira0437250-285 2014-07-01 17:14:36,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 1527 (ObjNum 1638 snap 422 seq_num 1638) dirty exists 2014-07-01 17:14:36,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: writing mira0435601-11 from 8949760 to 9633792 tid 3 2014-07-01 17:14:36,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: writing mira0435601-11 from 9633792 to 9986048 tid 4 2014-07-01 17:14:36,088.088 INFO:teuthology.task.rados.rados.0.mira043.stdout:3898: read oid 205 snap -1 2014-07-01 17:14:36,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: read oid 109 snap -1 2014-07-01 17:14:36,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: read oid 488 snap -1 2014-07-01 17:14:36,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 452 current snap is 0 2014-07-01 17:14:36,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: seq_num 2385 ranges {3162112=745472,3907584=557056,4464640=561152,5025792=86016} 2014-07-01 17:14:36,129.129 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: writing mira0435601-452 from 3162112 to 3907584 tid 1 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: writing mira0435601-452 from 3907584 to 4464640 tid 2 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: writing mira0435601-452 from 4464640 to 5025792 tid 3 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: writing mira0435601-452 from 5025792 to 5111808 tid 4 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3902: read oid 458 snap -1 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 184 current snap is 0 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: seq_num 2386 ranges {1720320=458752} 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: writing mira0435601-184 from 1720320 to 2179072 tid 1 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 118 current snap is 0 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: seq_num 2387 ranges {0=487424,487424=720896,1208320=483328} 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: writing mira0435601-118 from 0 to 487424 tid 1 2014-07-01 17:14:36,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: writing mira0435601-118 from 487424 to 1208320 tid 2 2014-07-01 17:14:36,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: writing mira0435601-118 from 1208320 to 1691648 tid 3 2014-07-01 17:14:36,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 58 current snap is 0 2014-07-01 17:14:36,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: seq_num 2388 ranges {6828032=516096} 2014-07-01 17:14:36,299.299 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: writing mira0435601-58 from 6828032 to 7344128 tid 1 2014-07-01 17:14:36,300.300 INFO:teuthology.task.rados.rados.0.mira043.stdout:3906: read oid 132 snap -1 2014-07-01 17:14:36,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 1 to mira0435601-237 2014-07-01 17:14:36,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 2 to mira0435601-237 2014-07-01 17:14:36,301.301 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 3 to mira0435601-237 2014-07-01 17:14:36,394.394 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 16 2014-07-01 17:14:36,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 4 to mira0435601-237 2014-07-01 17:14:36,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 6 to mira0435601-237 2014-07-01 17:14:36,395.395 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: expect (ObjNum 2094 snap 0 seq_num 2094) 2014-07-01 17:14:37,462.462 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.1. 2014-07-01 17:14:37,463.463 INFO:teuthology.task.thrashosds.thrasher:Added osd 1 2014-07-01 17:14:37,464.464 INFO:teuthology.task.rados.rados.0.mira043.stdout:3624: done (1 left) 2014-07-01 17:14:37,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:3626: done (0 left) 2014-07-01 17:14:37,465.465 INFO:teuthology.task.rados.rados.0.mira043.stdout:3627: snap_remove snap 375 2014-07-01 17:14:37,759.759 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: finishing write tid 7 to mira0435601-237 2014-07-01 17:14:37,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 141 (ObjNum 2378 snap 0 seq_num 2378) dirty exists 2014-07-01 17:14:37,760.760 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: expect (ObjNum 1835 snap 0 seq_num 1835) 2014-07-01 17:14:38,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3627: done (0 left) 2014-07-01 17:14:38,782.782 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: snap_remove snap 371 2014-07-01 17:14:39,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:3628: done (0 left) 2014-07-01 17:14:39,787.787 INFO:teuthology.task.rados.rados.0.mira043.stdout:3629: snap_create 2014-07-01 17:14:40,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:3629: done (0 left) 2014-07-01 17:14:40,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: snap_remove snap 418 2014-07-01 17:14:41,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: expect (ObjNum 2370 snap 0 seq_num 2370) 2014-07-01 17:14:41,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3630: done (0 left) 2014-07-01 17:14:41,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: write oid 14 current snap is 425 2014-07-01 17:14:41,801.801 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: seq_num 1639 ranges {673799=729046,2046638=227161} 2014-07-01 17:14:41,825.825 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: writing mira0437250-14 from 673799 to 1402845 tid 1 2014-07-01 17:14:41,838.838 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: writing mira0437250-14 from 2046638 to 2273799 tid 2 2014-07-01 17:14:41,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: read oid 43 snap -1 2014-07-01 17:14:41,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3633: delete oid 261 current snap is 425 2014-07-01 17:14:41,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3632: done (2 left) 2014-07-01 17:14:41,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3633: done (1 left) 2014-07-01 17:14:41,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:3634: rollback oid 292 current snap is 425 2014-07-01 17:14:41,861.861 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 292 to 410 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3635: read oid 48 snap 394 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3636: read oid 12 snap -1 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: read oid 108 snap -1 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: write oid 481 current snap is 425 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: seq_num 1641 ranges {581634=707727,1873878=307756} 2014-07-01 17:14:41,971.971 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: writing mira0437250-481 from 581634 to 1289361 tid 1 2014-07-01 17:14:41,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: writing mira0437250-481 from 1873878 to 2181634 tid 2 2014-07-01 17:14:41,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: write oid 403 current snap is 425 2014-07-01 17:14:41,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: seq_num 1642 ranges {775040=679531,2200490=584063,3409594=424660,3975039=1} 2014-07-01 17:14:41,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0437250-403 from 775040 to 1454571 tid 1 2014-07-01 17:14:41,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0437250-403 from 2200490 to 2784553 tid 2 2014-07-01 17:14:41,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0437250-403 from 3409594 to 3834254 tid 3 2014-07-01 17:14:41,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: writing mira0437250-403 from 3975039 to 3975040 tid 4 2014-07-01 17:14:41,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: write oid 212 current snap is 425 2014-07-01 17:14:41,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: seq_num 1643 ranges {471294=403783,1515646=654259,2812868=407579,3671293=1} 2014-07-01 17:14:41,997.997 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: writing mira0437250-212 from 471294 to 875077 tid 1 2014-07-01 17:14:42,017.017 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: writing mira0437250-212 from 1515646 to 2169905 tid 2 2014-07-01 17:14:42,034.034 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: writing mira0437250-212 from 2812868 to 3220447 tid 3 2014-07-01 17:14:42,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: writing mira0437250-212 from 3671293 to 3671294 tid 4 2014-07-01 17:14:42,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3641: snap_create 2014-07-01 17:14:42,143.143 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: finishing write tid 1 to mira0437250-14 2014-07-01 17:14:42,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: finishing write tid 2 to mira0437250-14 2014-07-01 17:14:42,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: finishing write tid 4 to mira0437250-14 2014-07-01 17:14:42,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: finishing write tid 5 to mira0437250-14 2014-07-01 17:14:42,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 14 v 662 (ObjNum 1639 snap 425 seq_num 1639) dirty exists 2014-07-01 17:14:42,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: expect (ObjNum 107 snap 0 seq_num 107) 2014-07-01 17:14:42,461.461 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:42,461.461 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:42,462.462 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool data 2014-07-01 17:14:42,462.462 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:14:43,048.048 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:14:43,753.753 INFO:teuthology.task.rados.rados.0.mira043.stdout:3635: expect (ObjNum 47 snap 0 seq_num 47) 2014-07-01 17:14:44,072.072 INFO:teuthology.task.rados.rados.0.mira043.stdout:3906: expect (ObjNum 131 snap 0 seq_num 131) 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: finishing write tid 1 to mira0435601-415 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: finishing write tid 1 to mira0435601-184 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: finishing write tid 3 to mira0435601-184 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: finishing write tid 1 to mira0435601-118 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: finishing write tid 2 to mira0435601-118 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: finishing write tid 3 to mira0435601-118 2014-07-01 17:14:44,264.264 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: finishing write tid 5 to mira0435601-118 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: finishing write tid 6 to mira0435601-118 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 118 v 336 (ObjNum 2387 snap 0 seq_num 2387) dirty exists 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: done (15 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3891: done (14 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: done (13 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: done (12 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: done (11 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3902: done (10 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: done (9 left) 2014-07-01 17:14:44,265.265 INFO:teuthology.task.rados.rados.0.mira043.stdout:3906: done (8 left) 2014-07-01 17:14:44,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: read oid 359 snap -1 2014-07-01 17:14:44,266.266 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 1 to mira0435601-58 2014-07-01 17:14:44,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 3 to mira0435601-58 2014-07-01 17:14:44,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: finishing write tid 4 to mira0435601-184 2014-07-01 17:14:44,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 184 v 223 (ObjNum 2386 snap 0 seq_num 2386) dirty exists 2014-07-01 17:14:44,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 4 to mira0435601-58 2014-07-01 17:14:44,267.267 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 58 v 170 (ObjNum 2388 snap 0 seq_num 2388) dirty exists 2014-07-01 17:14:44,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 1 to mira0435601-11 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: finishing write tid 1 to mira0435601-396 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 2 to mira0435601-11 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: finishing write tid 2 to mira0435601-396 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: finishing write tid 3 to mira0435601-396 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: done (8 left) 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: done (7 left) 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 53 current snap is 0 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: seq_num 2389 ranges {692224=630784,1323008=536576,1859584=266240} 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0435601-53 from 692224 to 1323008 tid 1 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0435601-53 from 1323008 to 1859584 tid 2 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0435601-53 from 1859584 to 2125824 tid 3 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 419 current snap is 0 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: seq_num 2390 ranges {0=708608,708608=401408} 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: writing mira0435601-419 from 0 to 708608 tid 1 2014-07-01 17:14:44,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: writing mira0435601-419 from 708608 to 1110016 tid 2 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3910: delete oid 196 current snap is 0 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 3 to mira0435601-11 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 4 to mira0435601-11 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: finishing write tid 5 to mira0435601-396 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 6 to mira0435601-11 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 1 to mira0435601-452 2014-07-01 17:14:44,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 2 to mira0435601-452 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 3 to mira0435601-452 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 4 to mira0435601-452 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 6 to mira0435601-452 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: finishing write tid 7 to mira0435601-11 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 220 (ObjNum 2384 snap 0 seq_num 2384) dirty exists 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: finishing write tid 6 to mira0435601-396 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 396 v 331 (ObjNum 2380 snap 0 seq_num 2380) dirty exists 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: finishing write tid 7 to mira0435601-452 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 452 v 336 (ObjNum 2385 snap 0 seq_num 2385) dirty exists 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: finishing write tid 3 to mira0435601-415 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: finishing write tid 4 to mira0435601-415 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 415 v 304 (ObjNum 2379 snap 0 seq_num 2379) dirty exists 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: finishing write tid 1 to mira0435601-292 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: finishing write tid 2 to mira0435601-292 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: finishing write tid 4 to mira0435601-292 2014-07-01 17:14:44,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3898: expect (ObjNum 204 snap 0 seq_num 204) 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: finishing write tid 1 to mira0437250-481 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: finishing write tid 2 to mira0437250-481 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: finishing write tid 4 to mira0437250-481 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: finishing write tid 5 to mira0437250-481 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 1318 (ObjNum 1641 snap 425 seq_num 1641) dirty exists 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 1 to mira0437250-403 2014-07-01 17:14:44,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3631: done (8 left) 2014-07-01 17:14:44,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3635: done (7 left) 2014-07-01 17:14:44,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3636: done (6 left) 2014-07-01 17:14:44,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3637: done (5 left) 2014-07-01 17:14:44,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3638: done (4 left) 2014-07-01 17:14:44,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3641: done (3 left) 2014-07-01 17:14:44,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: rollback oid 277 current snap is 426 2014-07-01 17:14:44,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 277 to 373 2014-07-01 17:14:45,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 2 to mira0437250-403 2014-07-01 17:14:45,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 3 to mira0437250-403 2014-07-01 17:14:45,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 4 to mira0437250-403 2014-07-01 17:14:45,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 6 to mira0437250-403 2014-07-01 17:14:45,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: finishing write tid 7 to mira0437250-403 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 403 v 1818 (ObjNum 1642 snap 425 seq_num 1642) dirty exists 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 1 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 2 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 3 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 4 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3639: done (3 left) 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3643: snap_remove snap 425 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 6 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 1811 (ObjNum 1435 snap 342 seq_num 1435) dirty exists 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: finishing write tid 7 to mira0437250-212 2014-07-01 17:14:45,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 1164 (ObjNum 1643 snap 425 seq_num 1643) dirty exists 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: finishing write tid 5 to mira0435601-292 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 307 (ObjNum 2383 snap 0 seq_num 2383) dirty exists 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: done (9 left) 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: done (8 left) 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: done (7 left) 2014-07-01 17:14:45,524.524 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: done (6 left) 2014-07-01 17:14:45,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3898: done (5 left) 2014-07-01 17:14:45,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: done (4 left) 2014-07-01 17:14:45,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3910: done (3 left) 2014-07-01 17:14:45,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 124 current snap is 0 2014-07-01 17:14:45,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: seq_num 2392 ranges {3911680=606208,4517888=532480,5050368=626688,5677056=446464} 2014-07-01 17:14:45,560.560 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: writing mira0435601-124 from 3911680 to 4517888 tid 1 2014-07-01 17:14:45,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: writing mira0435601-124 from 4517888 to 5050368 tid 2 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 277 v 1012 (ObjNum 1360 snap 309 seq_num 1360) dirty exists 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: writing mira0435601-124 from 5050368 to 5677056 tid 3 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: writing mira0435601-124 from 5677056 to 6123520 tid 4 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3912: read oid 298 snap -1 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 1 to mira0435601-53 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 2 to mira0435601-53 2014-07-01 17:14:45,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 3 to mira0435601-53 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 5 to mira0435601-53 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 6 to mira0435601-53 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 53 v 247 (ObjNum 2389 snap 0 seq_num 2389) dirty exists 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: finishing write tid 1 to mira0435601-419 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: finishing write tid 2 to mira0435601-419 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: finishing write tid 4 to mira0435601-419 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: finishing write tid 5 to mira0435601-419 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 310 (ObjNum 2390 snap 0 seq_num 2390) dirty exists 2014-07-01 17:14:45,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: expect (ObjNum 1542 snap 0 seq_num 1542) 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3634: done (3 left) 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3640: done (2 left) 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3642: done (1 left) 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3643: done (0 left) 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: read oid 198 snap -1 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3645: rollback oid 467 current snap is 426 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 467 to 361 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3646: rollback oid 394 current snap is 426 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 394 to 420 2014-07-01 17:14:45,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3647: rollback oid 166 current snap is 426 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 166 to 384 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: read oid 470 snap 402 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3649: snap_create 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 394 v 1695 (ObjNum 990 snap 188 seq_num 990) dirty exists 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 166 v 766 (ObjNum 1169 snap 246 seq_num 1169) dirty exists 2014-07-01 17:14:45,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: expect (ObjNum 938 snap 171 seq_num 938) 2014-07-01 17:14:46,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 1539 (ObjNum 1409 snap 328 seq_num 1409) dirty exists 2014-07-01 17:14:46,939.939 INFO:teuthology.task.rados.rados.0.mira043.stdout:3644: done (5 left) 2014-07-01 17:14:46,940.940 INFO:teuthology.task.rados.rados.0.mira043.stdout:3645: done (4 left) 2014-07-01 17:14:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3646: done (3 left) 2014-07-01 17:14:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3647: done (2 left) 2014-07-01 17:14:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3648: done (1 left) 2014-07-01 17:14:46,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3649: done (0 left) 2014-07-01 17:14:46,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: write oid 141 current snap is 427 2014-07-01 17:14:46,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: seq_num 1644 ranges {566524=775588,1745107=724454,3261646=104878} 2014-07-01 17:14:46,964.964 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: writing mira0437250-141 from 566524 to 1342112 tid 1 2014-07-01 17:14:46,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: writing mira0437250-141 from 1745107 to 2469561 tid 2 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: writing mira0437250-141 from 3261646 to 3366524 tid 3 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3651: read oid 97 snap 394 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: finishing write tid 1 to mira0437250-141 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3652: rollback oid 236 current snap is 427 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 236 to 357 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: rollback oid 352 current snap is 427 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 352 to 392 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3654: delete oid 471 current snap is 427 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 352 v 0 (ObjNum 320 snap 0 seq_num 4294967295) dirty dne 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3651: done (4 left) 2014-07-01 17:14:47,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3653: done (3 left) 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3654: done (2 left) 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: read oid 28 snap 417 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: write oid 291 current snap is 427 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: seq_num 1646 ranges {571511=755292,1371510=1} 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: writing mira0437250-291 from 571511 to 1326803 tid 1 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: writing mira0437250-291 from 1371510 to 1371511 tid 2 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: snap_remove snap 355 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 236 v 828 (ObjNum 1319 snap 302 seq_num 1319) dirty exists 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: finishing write tid 2 to mira0437250-141 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: finishing write tid 3 to mira0437250-141 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: finishing write tid 5 to mira0437250-141 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: finishing write tid 6 to mira0437250-141 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 1104 (ObjNum 1644 snap 427 seq_num 1644) dirty exists 2014-07-01 17:14:47,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: expect (ObjNum 27 snap 0 seq_num 27) 2014-07-01 17:14:47,264.264 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pgp_num 34' 2014-07-01 17:14:47,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: finishing write tid 1 to mira0437250-291 2014-07-01 17:14:47,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: finishing write tid 2 to mira0437250-291 2014-07-01 17:14:47,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: finishing write tid 4 to mira0437250-291 2014-07-01 17:14:47,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: finishing write tid 5 to mira0437250-291 2014-07-01 17:14:47,427.427 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 1673 (ObjNum 1646 snap 427 seq_num 1646) dirty exists 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3650: done (4 left) 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3652: done (3 left) 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3655: done (2 left) 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3656: done (1 left) 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3657: done (0 left) 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: write oid 28 current snap is 427 2014-07-01 17:14:47,686.686 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: seq_num 1647 ranges {516341=478521,1501007=615334} 2014-07-01 17:14:47,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: writing mira0437250-28 from 516341 to 994862 tid 1 2014-07-01 17:14:47,725.725 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: writing mira0437250-28 from 1501007 to 2116341 tid 2 2014-07-01 17:14:47,726.726 INFO:teuthology.task.rados.rados.0.mira043.stdout:3659: snap_remove snap 384 2014-07-01 17:14:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: finishing write tid 1 to mira0437250-28 2014-07-01 17:14:47,865.865 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: finishing write tid 2 to mira0437250-28 2014-07-01 17:14:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: finishing write tid 4 to mira0437250-28 2014-07-01 17:14:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: finishing write tid 5 to mira0437250-28 2014-07-01 17:14:47,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 28 v 1759 (ObjNum 1647 snap 427 seq_num 1647) dirty exists 2014-07-01 17:14:48,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3658: done (1 left) 2014-07-01 17:14:48,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3659: done (0 left) 2014-07-01 17:14:48,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3660: snap_create 2014-07-01 17:14:48,721.721 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pgp_num to 34 2014-07-01 17:14:49,699.699 INFO:teuthology.task.rados.rados.0.mira043.stdout:3660: done (0 left) 2014-07-01 17:14:49,700.700 INFO:teuthology.task.rados.rados.0.mira043.stdout:3661: delete oid 256 current snap is 428 2014-07-01 17:14:49,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3661: done (0 left) 2014-07-01 17:14:49,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3662: read oid 373 snap 402 2014-07-01 17:14:49,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: read oid 423 snap -1 2014-07-01 17:14:49,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: read oid 172 snap -1 2014-07-01 17:14:49,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: snap_remove snap 394 2014-07-01 17:14:49,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: expect (ObjNum 1381 snap 313 seq_num 1381) 2014-07-01 17:14:49,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3662: expect (ObjNum 372 snap 0 seq_num 372) 2014-07-01 17:14:49,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 1 to mira0435601-124 2014-07-01 17:14:49,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 2 to mira0435601-124 2014-07-01 17:14:49,882.882 INFO:teuthology.task.rados.rados.0.mira043.stdout:3912: expect (ObjNum 2367 snap 0 seq_num 2367) 2014-07-01 17:14:50,141.141 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: expect (ObjNum 1481 snap 363 seq_num 1481) 2014-07-01 17:14:50,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 3 to mira0435601-124 2014-07-01 17:14:50,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 4 to mira0435601-124 2014-07-01 17:14:50,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 6 to mira0435601-124 2014-07-01 17:14:50,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: finishing write tid 7 to mira0435601-124 2014-07-01 17:14:50,212.212 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 124 v 360 (ObjNum 2392 snap 0 seq_num 2392) dirty exists 2014-07-01 17:14:50,213.213 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: done (4 left) 2014-07-01 17:14:50,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: done (3 left) 2014-07-01 17:14:50,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: done (2 left) 2014-07-01 17:14:50,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: done (1 left) 2014-07-01 17:14:50,214.214 INFO:teuthology.task.rados.rados.0.mira043.stdout:3912: done (0 left) 2014-07-01 17:14:50,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:3913: read oid 124 snap -1 2014-07-01 17:14:50,215.215 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 313 current snap is 0 2014-07-01 17:14:50,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: seq_num 2393 ranges {11505664=479232,11984896=516096,12500992=454656,12955648=589824,13545472=45056} 2014-07-01 17:14:50,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: writing mira0435601-313 from 11505664 to 11984896 tid 1 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: writing mira0435601-313 from 11984896 to 12500992 tid 2 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: writing mira0435601-313 from 12500992 to 12955648 tid 3 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: writing mira0435601-313 from 12955648 to 13545472 tid 4 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: writing mira0435601-313 from 13545472 to 13590528 tid 5 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3915: read oid 396 snap -1 2014-07-01 17:14:50,400.400 INFO:teuthology.task.rados.rados.0.mira043.stdout:3913: expect (ObjNum 2392 snap 0 seq_num 2392) 2014-07-01 17:14:50,719.719 INFO:teuthology.task.rados.rados.0.mira043.stdout:3662: done (3 left) 2014-07-01 17:14:50,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:3663: done (2 left) 2014-07-01 17:14:50,720.720 INFO:teuthology.task.rados.rados.0.mira043.stdout:3664: done (1 left) 2014-07-01 17:14:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3665: done (0 left) 2014-07-01 17:14:50,721.721 INFO:teuthology.task.rados.rados.0.mira043.stdout:3666: delete oid 283 current snap is 428 2014-07-01 17:14:50,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:3666: done (0 left) 2014-07-01 17:14:50,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: delete oid 400 current snap is 428 2014-07-01 17:14:50,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3667: done (0 left) 2014-07-01 17:14:50,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3668: snap_remove snap 304 2014-07-01 17:14:51,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3668: done (0 left) 2014-07-01 17:14:51,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3669: rollback oid 377 current snap is 428 2014-07-01 17:14:51,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 377 to 415 2014-07-01 17:14:51,729.729 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: delete oid 164 current snap is 428 2014-07-01 17:14:51,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3670: done (1 left) 2014-07-01 17:14:51,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: write oid 309 current snap is 428 2014-07-01 17:14:51,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: seq_num 1652 ranges {256616=1} 2014-07-01 17:14:51,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: writing mira0437250-309 from 256616 to 256617 tid 1 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: read oid 258 snap -1 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: snap_remove snap 402 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 377 v 1228 (ObjNum 1535 snap 388 seq_num 1535) dirty exists 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 1 to mira0437250-309 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 3 to mira0437250-309 2014-07-01 17:14:51,895.895 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: expect (ObjNum 257 snap 0 seq_num 257) 2014-07-01 17:14:52,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: finishing write tid 4 to mira0437250-309 2014-07-01 17:14:52,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 309 v 779 (ObjNum 1652 snap 428 seq_num 1652) dirty exists 2014-07-01 17:14:52,774.774 INFO:teuthology.task.rados.rados.0.mira043.stdout:3669: done (3 left) 2014-07-01 17:14:52,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3671: done (2 left) 2014-07-01 17:14:52,775.775 INFO:teuthology.task.rados.rados.0.mira043.stdout:3672: done (1 left) 2014-07-01 17:14:52,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3673: done (0 left) 2014-07-01 17:14:52,776.776 INFO:teuthology.task.rados.rados.0.mira043.stdout:3674: delete oid 137 current snap is 428 2014-07-01 17:14:52,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3674: done (0 left) 2014-07-01 17:14:52,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: rollback oid 359 current snap is 428 2014-07-01 17:14:52,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 359 to 419 2014-07-01 17:14:52,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: read oid 363 snap -1 2014-07-01 17:14:52,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: write oid 346 current snap is 428 2014-07-01 17:14:52,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: seq_num 1654 ranges {696676=1} 2014-07-01 17:14:52,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: writing mira0437250-346 from 696676 to 696677 tid 1 2014-07-01 17:14:52,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: snap_remove snap 403 2014-07-01 17:14:52,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 359 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:14:52,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: finishing write tid 1 to mira0437250-346 2014-07-01 17:14:52,941.941 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: expect (ObjNum 362 snap 0 seq_num 362) 2014-07-01 17:14:53,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: finishing write tid 3 to mira0437250-346 2014-07-01 17:14:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: finishing write tid 4 to mira0437250-346 2014-07-01 17:14:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 1236 (ObjNum 1654 snap 428 seq_num 1654) dirty exists 2014-07-01 17:14:53,733.733 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:14:53,733.733 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:14:53,733.733 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool rbd 2014-07-01 17:14:53,734.734 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:14:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3675: done (3 left) 2014-07-01 17:14:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3676: done (2 left) 2014-07-01 17:14:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3677: done (1 left) 2014-07-01 17:14:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3678: done (0 left) 2014-07-01 17:14:53,784.784 INFO:teuthology.task.rados.rados.0.mira043.stdout:3679: snap_remove snap 413 2014-07-01 17:14:54,475.475 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 1 to mira0435601-313 2014-07-01 17:14:54,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 2 to mira0435601-313 2014-07-01 17:14:54,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 3 to mira0435601-313 2014-07-01 17:14:54,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 4 to mira0435601-313 2014-07-01 17:14:54,477.477 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 5 to mira0435601-313 2014-07-01 17:14:54,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 7 to mira0435601-313 2014-07-01 17:14:54,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: finishing write tid 8 to mira0435601-313 2014-07-01 17:14:54,478.478 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 248 (ObjNum 2393 snap 0 seq_num 2393) dirty exists 2014-07-01 17:14:54,479.479 INFO:teuthology.task.rados.rados.0.mira043.stdout:3915: expect (ObjNum 2380 snap 0 seq_num 2380) 2014-07-01 17:14:54,543.543 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:14:54,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:3679: done (0 left) 2014-07-01 17:14:54,807.807 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: snap_create 2014-07-01 17:14:55,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3680: done (0 left) 2014-07-01 17:14:55,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: write oid 310 current snap is 429 2014-07-01 17:14:55,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: seq_num 1655 ranges {757859=477595,1957858=1} 2014-07-01 17:14:55,828.828 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: writing mira0437250-310 from 757859 to 1235454 tid 1 2014-07-01 17:14:55,833.833 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: writing mira0437250-310 from 1957858 to 1957859 tid 2 2014-07-01 17:14:55,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: write oid 465 current snap is 429 2014-07-01 17:14:55,834.834 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: seq_num 1656 ranges {700640=449400,1564612=606540,2700639=1} 2014-07-01 17:14:55,851.851 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0437250-465 from 700640 to 1150040 tid 1 2014-07-01 17:14:55,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0437250-465 from 1564612 to 2171152 tid 2 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: writing mira0437250-465 from 2700639 to 2700640 tid 3 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: write oid 446 current snap is 429 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: seq_num 1657 ranges {796894=457300,1831192=408959,2396893=1} 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: writing mira0437250-446 from 796894 to 1254194 tid 1 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: writing mira0437250-446 from 1831192 to 2240151 tid 2 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: writing mira0437250-446 from 2396893 to 2396894 tid 3 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: read oid 289 snap -1 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: finishing write tid 1 to mira0437250-310 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: finishing write tid 2 to mira0437250-310 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: finishing write tid 4 to mira0437250-310 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: finishing write tid 5 to mira0437250-310 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 846 (ObjNum 1655 snap 429 seq_num 1655) dirty exists 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 1 to mira0437250-465 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 2 to mira0437250-465 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 3 to mira0437250-465 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 5 to mira0437250-465 2014-07-01 17:14:55,995.995 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: finishing write tid 6 to mira0437250-465 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 465 v 1046 (ObjNum 1656 snap 429 seq_num 1656) dirty exists 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3681: done (3 left) 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3682: done (2 left) 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3685: delete oid 109 current snap is 429 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 1 to mira0437250-446 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3685: done (2 left) 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3686: snap_create 2014-07-01 17:14:55,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: expect (ObjNum 1490 snap 366 seq_num 1490) 2014-07-01 17:14:56,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 2 to mira0437250-446 2014-07-01 17:14:56,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 3 to mira0437250-446 2014-07-01 17:14:56,113.113 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 5 to mira0437250-446 2014-07-01 17:14:56,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: finishing write tid 6 to mira0437250-446 2014-07-01 17:14:56,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 446 v 1820 (ObjNum 1657 snap 429 seq_num 1657) dirty exists 2014-07-01 17:14:56,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:3683: done (2 left) 2014-07-01 17:14:56,818.818 INFO:teuthology.task.rados.rados.0.mira043.stdout:3684: done (1 left) 2014-07-01 17:14:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3686: done (0 left) 2014-07-01 17:14:56,819.819 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: snap_create 2014-07-01 17:14:57,687.687 INFO:teuthology.task.rados.rados.0.mira043.stdout:3913: done (2 left) 2014-07-01 17:14:57,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: done (1 left) 2014-07-01 17:14:57,689.689 INFO:teuthology.task.rados.rados.0.mira043.stdout:3915: done (0 left) 2014-07-01 17:14:57,691.691 INFO:teuthology.task.rados.rados.0.mira043.stdout:3916: read oid 390 snap -1 2014-07-01 17:14:57,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: read oid 281 snap -1 2014-07-01 17:14:57,692.692 INFO:teuthology.task.rados.rados.0.mira043.stdout:3918: read oid 432 snap -1 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 305 current snap is 0 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: seq_num 2394 ranges {1568768=557056,2125824=544768,2670592=659456} 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: writing mira0435601-305 from 1568768 to 2125824 tid 1 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: writing mira0435601-305 from 2125824 to 2670592 tid 2 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: writing mira0435601-305 from 2670592 to 3330048 tid 3 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: read oid 288 snap -1 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: expect (ObjNum 2350 snap 0 seq_num 2350) 2014-07-01 17:14:57,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3687: done (0 left) 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: write oid 123 current snap is 431 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: seq_num 1659 ranges {401881=1} 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: writing mira0437250-123 from 401881 to 401882 tid 1 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: write oid 473 current snap is 431 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: seq_num 1660 ranges {98134=1} 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: writing mira0437250-473 from 98134 to 98135 tid 1 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: write oid 405 current snap is 431 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: seq_num 1661 ranges {441941=400000} 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: writing mira0437250-405 from 441941 to 841941 tid 1 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: read oid 19 snap -1 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 1 to mira0437250-473 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3692: snap_create 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 3 to mira0437250-473 2014-07-01 17:14:57,858.858 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: finishing write tid 4 to mira0437250-473 2014-07-01 17:14:57,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 473 v 651 (ObjNum 1660 snap 431 seq_num 1660) dirty exists 2014-07-01 17:14:57,859.859 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: finishing write tid 1 to mira0437250-123 2014-07-01 17:14:57,862.862 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: finishing write tid 3 to mira0437250-123 2014-07-01 17:14:57,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: finishing write tid 4 to mira0437250-123 2014-07-01 17:14:57,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 123 v 1208 (ObjNum 1659 snap 431 seq_num 1659) dirty exists 2014-07-01 17:14:57,866.866 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 1 to mira0437250-405 2014-07-01 17:14:58,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 3 to mira0437250-405 2014-07-01 17:14:58,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: finishing write tid 4 to mira0437250-405 2014-07-01 17:14:58,021.021 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 405 v 1048 (ObjNum 1661 snap 431 seq_num 1661) dirty exists 2014-07-01 17:14:58,022.022 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: expect (ObjNum 1172 snap 246 seq_num 1172) 2014-07-01 17:14:58,115.115 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set rbd pgp_num 64' 2014-07-01 17:14:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: finishing write tid 1 to mira0435601-305 2014-07-01 17:14:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: finishing write tid 2 to mira0435601-305 2014-07-01 17:14:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: finishing write tid 3 to mira0435601-305 2014-07-01 17:14:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: finishing write tid 5 to mira0435601-305 2014-07-01 17:14:58,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: finishing write tid 6 to mira0435601-305 2014-07-01 17:14:58,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 305 v 413 (ObjNum 2394 snap 0 seq_num 2394) dirty exists 2014-07-01 17:14:58,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: expect (ObjNum 2334 snap 0 seq_num 2334) 2014-07-01 17:14:58,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3688: done (4 left) 2014-07-01 17:14:58,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3689: done (3 left) 2014-07-01 17:14:58,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3690: done (2 left) 2014-07-01 17:14:58,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3691: done (1 left) 2014-07-01 17:14:58,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3692: done (0 left) 2014-07-01 17:14:58,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: snap_create 2014-07-01 17:14:58,869.869 INFO:teuthology.orchestra.run.mira079.stderr:set pool 2 pgp_num to 64 2014-07-01 17:14:59,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3918: expect (ObjNum 1407 snap 0 seq_num 1407) 2014-07-01 17:15:02,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3916: done (4 left) 2014-07-01 17:15:02,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: done (3 left) 2014-07-01 17:15:02,420.420 INFO:teuthology.task.rados.rados.0.mira043.stdout:3918: done (2 left) 2014-07-01 17:15:02,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: done (1 left) 2014-07-01 17:15:02,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: done (0 left) 2014-07-01 17:15:02,421.421 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: read oid 403 snap -1 2014-07-01 17:15:02,422.422 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: read oid 142 snap -1 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: read oid 468 snap -1 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 288 current snap is 0 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: seq_num 2395 ranges {1335296=425984,1761280=528384,2289664=675840} 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: writing mira0435601-288 from 1335296 to 1761280 tid 1 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: writing mira0435601-288 from 1761280 to 2289664 tid 2 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: writing mira0435601-288 from 2289664 to 2965504 tid 3 2014-07-01 17:15:02,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: read oid 380 snap -1 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: done (4 left) 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 209 current snap is 0 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: seq_num 2396 ranges {3432448=700416,4132864=757760,4890624=45056} 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: writing mira0435601-209 from 3432448 to 4132864 tid 1 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: writing mira0435601-209 from 4132864 to 4890624 tid 2 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: writing mira0435601-209 from 4890624 to 4935680 tid 3 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3927: read oid 439 snap -1 2014-07-01 17:15:02,586.586 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: expect (ObjNum 1786 snap 0 seq_num 1786) 2014-07-01 17:15:03,879.879 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:03,880.880 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:15:03,880.880 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:15:03,881.881 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:15:04,322.322 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: expect (ObjNum 379 snap 0 seq_num 379) 2014-07-01 17:15:05,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: finishing write tid 1 to mira0435601-209 2014-07-01 17:15:05,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: expect (ObjNum 2371 snap 0 seq_num 2371) 2014-07-01 17:15:07,671.671 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: finishing write tid 2 to mira0435601-209 2014-07-01 17:15:07,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: finishing write tid 3 to mira0435601-209 2014-07-01 17:15:07,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: finishing write tid 5 to mira0435601-209 2014-07-01 17:15:07,672.672 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: finishing write tid 6 to mira0435601-209 2014-07-01 17:15:07,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 209 v 182 (ObjNum 2396 snap 0 seq_num 2396) dirty exists 2014-07-01 17:15:07,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3927: expect (ObjNum 1923 snap 0 seq_num 1923) 2014-07-01 17:15:08,300.300 INFO:teuthology.task.ceph.mon.b.mira079.stderr:2014-07-01 08:15:08.212064 7ff07a1ad700 -1 mon.b@2(peon).paxos(paxos updating c 6527..7163) lease_expire from mon.0 10.214.132.120:6789/0 is 4.455466 seconds in the past; mons are probably laggy (or possibly clocks are too skewed) 2014-07-01 17:15:08,301.301 INFO:teuthology.task.ceph.mon.a.mira079.stderr:2014-07-01 08:15:08.211607 7f75f718c700 -1 mon.a@1(peon).paxos(paxos updating c 6527..7163) lease_expire from mon.0 10.214.132.120:6789/0 is 4.455009 seconds in the past; mons are probably laggy (or possibly clocks are too skewed) 2014-07-01 17:15:08,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3693: done (0 left) 2014-07-01 17:15:08,316.316 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: write oid 489 current snap is 433 2014-07-01 17:15:08,317.317 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: seq_num 1662 ranges {786771=752418,1586770=1} 2014-07-01 17:15:08,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: writing mira0437250-489 from 786771 to 1539189 tid 1 2014-07-01 17:15:08,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: writing mira0437250-489 from 1586770 to 1586771 tid 2 2014-07-01 17:15:08,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: write oid 419 current snap is 433 2014-07-01 17:15:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: seq_num 1663 ranges {483025=475646,1283024=1} 2014-07-01 17:15:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: writing mira0437250-419 from 483025 to 958671 tid 1 2014-07-01 17:15:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: writing mira0437250-419 from 1283024 to 1283025 tid 2 2014-07-01 17:15:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: read oid 10 snap -1 2014-07-01 17:15:08,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 1 to mira0437250-489 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 2 to mira0437250-489 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 4 to mira0437250-489 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: finishing write tid 5 to mira0437250-489 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 489 v 1547 (ObjNum 1662 snap 433 seq_num 1662) dirty exists 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3694: done (2 left) 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: write oid 251 current snap is 433 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: seq_num 1664 ranges {693582=448474,1493581=1} 2014-07-01 17:15:08,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: writing mira0437250-251 from 693582 to 1142056 tid 1 2014-07-01 17:15:08,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: writing mira0437250-251 from 1493581 to 1493582 tid 2 2014-07-01 17:15:08,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: write oid 158 current snap is 433 2014-07-01 17:15:08,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: seq_num 1665 ranges {486988=455351,1630265=624793,2790034=496954} 2014-07-01 17:15:08,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: writing mira0437250-158 from 486988 to 942339 tid 1 2014-07-01 17:15:08,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: writing mira0437250-158 from 1630265 to 2255058 tid 2 2014-07-01 17:15:08,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: writing mira0437250-158 from 2790034 to 3286988 tid 3 2014-07-01 17:15:08,636.636 INFO:teuthology.task.rados.rados.0.mira043.stdout:3699: read oid 23 snap -1 2014-07-01 17:15:08,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: expect (ObjNum 1400 snap 322 seq_num 1400) 2014-07-01 17:15:08,663.663 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: finishing write tid 1 to mira0437250-419 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: finishing write tid 2 to mira0437250-419 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: finishing write tid 4 to mira0437250-419 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: finishing write tid 5 to mira0437250-419 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 1288 (ObjNum 1663 snap 433 seq_num 1663) dirty exists 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 1 to mira0437250-251 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 2 to mira0437250-251 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 4 to mira0437250-251 2014-07-01 17:15:08,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: finishing write tid 5 to mira0437250-251 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 251 v 980 (ObjNum 1664 snap 433 seq_num 1664) dirty exists 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 1 to mira0437250-158 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 2 to mira0437250-158 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 3 to mira0437250-158 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 5 to mira0437250-158 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: finishing write tid 6 to mira0437250-158 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 158 v 837 (ObjNum 1665 snap 433 seq_num 1665) dirty exists 2014-07-01 17:15:08,842.842 INFO:teuthology.task.rados.rados.0.mira043.stdout:3699: expect (ObjNum 1615 snap 416 seq_num 1615) 2014-07-01 17:15:09,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3695: done (4 left) 2014-07-01 17:15:09,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3696: done (3 left) 2014-07-01 17:15:09,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3697: done (2 left) 2014-07-01 17:15:09,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3698: done (1 left) 2014-07-01 17:15:09,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3699: done (0 left) 2014-07-01 17:15:09,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: snap_remove snap 329 2014-07-01 17:15:10,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3700: done (0 left) 2014-07-01 17:15:10,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: rollback oid 343 current snap is 433 2014-07-01 17:15:10,350.350 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 343 to 422 2014-07-01 17:15:10,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: delete oid 480 current snap is 433 2014-07-01 17:15:10,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3702: done (1 left) 2014-07-01 17:15:10,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: write oid 249 current snap is 433 2014-07-01 17:15:10,419.419 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: seq_num 1667 ranges {678471=550383,1724256=400255,2678470=1} 2014-07-01 17:15:10,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: writing mira0437250-249 from 678471 to 1228854 tid 1 2014-07-01 17:15:10,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: writing mira0437250-249 from 1724256 to 2124511 tid 2 2014-07-01 17:15:10,516.516 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: writing mira0437250-249 from 2678470 to 2678471 tid 3 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: write oid 444 current snap is 433 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: seq_num 1668 ranges {588229=400000} 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: writing mira0437250-444 from 588229 to 988229 tid 1 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: read oid 117 snap -1 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 1 to mira0437250-249 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 343 v 1643 (ObjNum 645 snap 67 seq_num 645) dirty exists 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 2 to mira0437250-249 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 3 to mira0437250-249 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 5 to mira0437250-249 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: finishing write tid 6 to mira0437250-249 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 924 (ObjNum 1667 snap 433 seq_num 1667) dirty exists 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3701: done (3 left) 2014-07-01 17:15:10,517.517 INFO:teuthology.task.rados.rados.0.mira043.stdout:3703: done (2 left) 2014-07-01 17:15:10,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: read oid 456 snap -1 2014-07-01 17:15:10,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3707: delete oid 413 current snap is 433 2014-07-01 17:15:10,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3707: done (3 left) 2014-07-01 17:15:10,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:3708: rollback oid 252 current snap is 433 2014-07-01 17:15:10,520.520 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 252 to 432 2014-07-01 17:15:10,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: write oid 33 current snap is 433 2014-07-01 17:15:10,584.584 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: seq_num 1670 ranges {379712=1} 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: writing mira0437250-33 from 379712 to 379713 tid 1 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3710: snap_create 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 1 to mira0437250-33 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 3 to mira0437250-33 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: finishing write tid 4 to mira0437250-33 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 33 v 1201 (ObjNum 1670 snap 433 seq_num 1670) dirty exists 2014-07-01 17:15:10,585.585 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: expect (ObjNum 808 snap 130 seq_num 808) 2014-07-01 17:15:11,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 0 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:15:11,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: finishing write tid 1 to mira0437250-444 2014-07-01 17:15:11,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: finishing write tid 3 to mira0437250-444 2014-07-01 17:15:11,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: finishing write tid 4 to mira0437250-444 2014-07-01 17:15:11,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 444 v 1057 (ObjNum 1668 snap 433 seq_num 1668) dirty exists 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3704: done (5 left) 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3705: done (4 left) 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3706: done (3 left) 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3708: done (2 left) 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3709: done (1 left) 2014-07-01 17:15:11,705.705 INFO:teuthology.task.rados.rados.0.mira043.stdout:3710: done (0 left) 2014-07-01 17:15:11,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: write oid 415 current snap is 434 2014-07-01 17:15:11,706.706 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: seq_num 1671 ranges {572094=474721,1540017=432460,2172093=1} 2014-07-01 17:15:11,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: writing mira0437250-415 from 572094 to 1046815 tid 1 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: writing mira0437250-415 from 1540017 to 1972477 tid 2 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: writing mira0437250-415 from 2172093 to 2172094 tid 3 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: read oid 28 snap -1 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: finishing write tid 1 to mira0437250-415 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: write oid 249 current snap is 434 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: seq_num 1672 ranges {667324=446525,1769477=97847} 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: writing mira0437250-249 from 667324 to 1113849 tid 1 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: writing mira0437250-249 from 1769477 to 1867324 tid 2 2014-07-01 17:15:11,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: read oid 424 snap -1 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: finishing write tid 2 to mira0437250-415 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: finishing write tid 3 to mira0437250-415 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: finishing write tid 5 to mira0437250-415 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: finishing write tid 6 to mira0437250-415 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 415 v 1041 (ObjNum 1671 snap 434 seq_num 1671) dirty exists 2014-07-01 17:15:11,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: expect (ObjNum 1647 snap 427 seq_num 1647) 2014-07-01 17:15:12,435.435 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pgp_num 64' 2014-07-01 17:15:12,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 1 to mira0437250-249 2014-07-01 17:15:12,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 2 to mira0437250-249 2014-07-01 17:15:12,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 4 to mira0437250-249 2014-07-01 17:15:12,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3711: done (3 left) 2014-07-01 17:15:12,528.528 INFO:teuthology.task.rados.rados.0.mira043.stdout:3712: done (2 left) 2014-07-01 17:15:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3714: done (1 left) 2014-07-01 17:15:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3715: rollback oid 41 current snap is 434 2014-07-01 17:15:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 41 to 357 2014-07-01 17:15:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: finishing write tid 5 to mira0437250-249 2014-07-01 17:15:12,529.529 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 249 v 928 (ObjNum 1672 snap 434 seq_num 1672) dirty exists 2014-07-01 17:15:12,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3713: done (1 left) 2014-07-01 17:15:12,530.530 INFO:teuthology.task.rados.rados.0.mira043.stdout:3716: read oid 109 snap -1 2014-07-01 17:15:12,531.531 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: snap_create 2014-07-01 17:15:12,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 41 v 0 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:15:13,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3715: done (2 left) 2014-07-01 17:15:13,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3716: done (1 left) 2014-07-01 17:15:13,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3717: done (0 left) 2014-07-01 17:15:13,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: write oid 413 current snap is 435 2014-07-01 17:15:13,377.377 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: seq_num 1673 ranges {479930=453402,1634011=445919} 2014-07-01 17:15:13,394.394 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pgp_num to 64 2014-07-01 17:15:13,401.401 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: writing mira0437250-413 from 479930 to 933332 tid 1 2014-07-01 17:15:13,430.430 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: writing mira0437250-413 from 1634011 to 2079930 tid 2 2014-07-01 17:15:13,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3719: delete oid 204 current snap is 435 2014-07-01 17:15:13,434.434 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: finishing write tid 1 to mira0437250-413 2014-07-01 17:15:13,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3719: done (1 left) 2014-07-01 17:15:13,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: rollback oid 18 current snap is 435 2014-07-01 17:15:13,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 18 to 432 2014-07-01 17:15:13,571.571 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: write oid 102 current snap is 435 2014-07-01 17:15:13,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: seq_num 1675 ranges {768565=549458,2075778=747289,3279216=289349} 2014-07-01 17:15:13,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: writing mira0437250-102 from 768565 to 1318023 tid 1 2014-07-01 17:15:13,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: writing mira0437250-102 from 2075778 to 2823067 tid 2 2014-07-01 17:15:13,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: writing mira0437250-102 from 3279216 to 3568565 tid 3 2014-07-01 17:15:13,572.572 INFO:teuthology.task.rados.rados.0.mira043.stdout:3722: rollback oid 172 current snap is 435 2014-07-01 17:15:13,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 172 to 427 2014-07-01 17:15:13,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 18 v 0 (ObjNum 30 snap 0 seq_num 0) dirty dne 2014-07-01 17:15:13,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: finishing write tid 2 to mira0437250-413 2014-07-01 17:15:13,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: finishing write tid 4 to mira0437250-413 2014-07-01 17:15:13,573.573 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: finishing write tid 5 to mira0437250-413 2014-07-01 17:15:13,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 413 v 1041 (ObjNum 1673 snap 435 seq_num 1673) dirty exists 2014-07-01 17:15:13,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: finishing write tid 1 to mira0437250-102 2014-07-01 17:15:13,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:3718: done (3 left) 2014-07-01 17:15:13,574.574 INFO:teuthology.task.rados.rados.0.mira043.stdout:3720: done (2 left) 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: snap_remove snap 428 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: finishing write tid 2 to mira0437250-102 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 172 v 1834 (ObjNum 1381 snap 313 seq_num 1381) dirty exists 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: finishing write tid 3 to mira0437250-102 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: finishing write tid 5 to mira0437250-102 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: finishing write tid 6 to mira0437250-102 2014-07-01 17:15:13,575.575 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 102 v 1429 (ObjNum 1675 snap 435 seq_num 1675) dirty exists 2014-07-01 17:15:14,431.431 INFO:teuthology.task.rados.rados.0.mira043.stdout:3721: done (2 left) 2014-07-01 17:15:14,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3722: done (1 left) 2014-07-01 17:15:14,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3723: done (0 left) 2014-07-01 17:15:14,432.432 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: snap_create 2014-07-01 17:15:15,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3724: done (0 left) 2014-07-01 17:15:15,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3725: read oid 484 snap -1 2014-07-01 17:15:15,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3726: read oid 18 snap 407 2014-07-01 17:15:15,447.447 INFO:teuthology.task.rados.rados.0.mira043.stdout:3727: snap_create 2014-07-01 17:15:15,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3725: expect (ObjNum 737 snap 102 seq_num 737) 2014-07-01 17:15:16,342.342 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: finishing write tid 1 to mira0435601-288 2014-07-01 17:15:16,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: finishing write tid 2 to mira0435601-288 2014-07-01 17:15:16,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: finishing write tid 3 to mira0435601-288 2014-07-01 17:15:16,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: done (5 left) 2014-07-01 17:15:16,343.343 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: done (4 left) 2014-07-01 17:15:16,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: done (3 left) 2014-07-01 17:15:16,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: done (2 left) 2014-07-01 17:15:16,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3927: done (1 left) 2014-07-01 17:15:16,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 121 current snap is 0 2014-07-01 17:15:16,346.346 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: seq_num 2397 ranges {659456=323584} 2014-07-01 17:15:16,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: writing mira0435601-121 from 659456 to 983040 tid 1 2014-07-01 17:15:16,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3929: read oid 477 snap -1 2014-07-01 17:15:16,509.509 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: finishing write tid 5 to mira0435601-288 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: finishing write tid 6 to mira0435601-288 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 288 v 390 (ObjNum 2395 snap 0 seq_num 2395) dirty exists 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: done (2 left) 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 398 current snap is 0 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: seq_num 2398 ranges {3067904=196608} 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: writing mira0435601-398 from 3067904 to 3264512 tid 1 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: read oid 218 snap -1 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: finishing write tid 1 to mira0435601-121 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: finishing write tid 3 to mira0435601-121 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: finishing write tid 4 to mira0435601-121 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 121 v 388 (ObjNum 2397 snap 0 seq_num 2397) dirty exists 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: done (3 left) 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3929: done (2 left) 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 231 current snap is 0 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: seq_num 2399 ranges {4743168=520192,5263360=401408} 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: writing mira0435601-231 from 4743168 to 5263360 tid 1 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: writing mira0435601-231 from 5263360 to 5664768 tid 2 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 296 current snap is 0 2014-07-01 17:15:16,510.510 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: seq_num 2400 ranges {2228224=544768,2772992=737280,3510272=675840,4186112=192512} 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: writing mira0435601-296 from 2228224 to 2772992 tid 1 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: writing mira0435601-296 from 2772992 to 3510272 tid 2 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3725: done (2 left) 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3726: done (1 left) 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3727: done (0 left) 2014-07-01 17:15:16,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3728: snap_create 2014-07-01 17:15:16,512.512 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: writing mira0435601-296 from 3510272 to 4186112 tid 3 2014-07-01 17:15:16,518.518 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: writing mira0435601-296 from 4186112 to 4378624 tid 4 2014-07-01 17:15:16,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: read oid 497 snap -1 2014-07-01 17:15:16,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: finishing write tid 1 to mira0435601-398 2014-07-01 17:15:16,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: finishing write tid 3 to mira0435601-398 2014-07-01 17:15:16,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: expect (ObjNum 2060 snap 0 seq_num 2060) 2014-07-01 17:15:17,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3728: done (0 left) 2014-07-01 17:15:17,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: rollback oid 32 current snap is 438 2014-07-01 17:15:17,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 32 to 325 2014-07-01 17:15:17,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: read oid 19 snap -1 2014-07-01 17:15:17,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: snap_remove snap 385 2014-07-01 17:15:17,468.468 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 32 v 0 (ObjNum 33134216 snap 0 seq_num 34722448) dirty dne 2014-07-01 17:15:17,621.621 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: expect (ObjNum 1172 snap 246 seq_num 1172) 2014-07-01 17:15:17,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: finishing write tid 4 to mira0435601-398 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 110 (ObjNum 2398 snap 0 seq_num 2398) dirty exists 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: finishing write tid 1 to mira0435601-231 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: finishing write tid 2 to mira0435601-231 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: finishing write tid 4 to mira0435601-231 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: finishing write tid 5 to mira0435601-231 2014-07-01 17:15:17,904.904 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 187 (ObjNum 2399 snap 0 seq_num 2399) dirty exists 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 1 to mira0435601-296 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 2 to mira0435601-296 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 3 to mira0435601-296 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 4 to mira0435601-296 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 6 to mira0435601-296 2014-07-01 17:15:17,905.905 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: expect (ObjNum 1765 snap 0 seq_num 1765) 2014-07-01 17:15:18,404.404 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:18,404.404 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:15:18,404.404 INFO:teuthology.task.thrashosds.thrasher:Growing pool unique_pool_0 2014-07-01 17:15:18,404.404 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:15:18,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3729: done (2 left) 2014-07-01 17:15:18,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3730: done (1 left) 2014-07-01 17:15:18,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3731: done (0 left) 2014-07-01 17:15:18,463.463 INFO:teuthology.task.rados.rados.0.mira043.stdout:3732: delete oid 305 current snap is 438 2014-07-01 17:15:18,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3732: done (0 left) 2014-07-01 17:15:18,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: write oid 298 current snap is 438 2014-07-01 17:15:18,550.550 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: seq_num 1677 ranges {524976=1} 2014-07-01 17:15:18,553.553 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: writing mira0437250-298 from 524976 to 524977 tid 1 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: write oid 462 current snap is 438 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: seq_num 1678 ranges {469807=652391,1269806=1} 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: writing mira0437250-462 from 469807 to 1122198 tid 1 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: writing mira0437250-462 from 1269806 to 1269807 tid 2 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: snap_remove snap 426 2014-07-01 17:15:18,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: finishing write tid 1 to mira0437250-298 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: finishing write tid 3 to mira0437250-298 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: finishing write tid 4 to mira0437250-298 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 1312 (ObjNum 1677 snap 438 seq_num 1677) dirty exists 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: finishing write tid 1 to mira0437250-462 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: finishing write tid 2 to mira0437250-462 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: finishing write tid 4 to mira0437250-462 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: finishing write tid 5 to mira0437250-462 2014-07-01 17:15:18,631.631 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 1264 (ObjNum 1678 snap 438 seq_num 1678) dirty exists 2014-07-01 17:15:18,984.984 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:15:19,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3733: done (2 left) 2014-07-01 17:15:19,487.487 INFO:teuthology.task.rados.rados.0.mira043.stdout:3734: done (1 left) 2014-07-01 17:15:19,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3735: done (0 left) 2014-07-01 17:15:19,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: read oid 182 snap -1 2014-07-01 17:15:19,488.488 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: read oid 347 snap -1 2014-07-01 17:15:19,489.489 INFO:teuthology.task.rados.rados.0.mira043.stdout:3738: read oid 205 snap -1 2014-07-01 17:15:19,490.490 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: read oid 490 snap -1 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3740: read oid 179 snap 433 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: snap_remove snap 393 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: expect (ObjNum 1544 snap 392 seq_num 1544) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: finishing write tid 7 to mira0435601-296 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 296 v 247 (ObjNum 2400 snap 0 seq_num 2400) dirty exists 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: done (4 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: done (3 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: done (2 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: done (1 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: done (0 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3935: delete oid 87 current snap is 0 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3935: done (0 left) 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 295 current snap is 0 2014-07-01 17:15:19,657.657 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: seq_num 2402 ranges {4153344=692224,4845568=503808,5349376=622592,5971968=77824} 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: writing mira0435601-295 from 4153344 to 4845568 tid 1 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: writing mira0435601-295 from 4845568 to 5349376 tid 2 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: writing mira0435601-295 from 5349376 to 5971968 tid 3 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: writing mira0435601-295 from 5971968 to 6049792 tid 4 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3937: read oid 403 snap -1 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 1 to mira0435601-295 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 2 to mira0435601-295 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 314 current snap is 0 2014-07-01 17:15:19,658.658 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: seq_num 2403 ranges {741376=765952,1507328=487424,1994752=315392} 2014-07-01 17:15:19,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: writing mira0435601-314 from 741376 to 1507328 tid 1 2014-07-01 17:15:19,711.711 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: writing mira0435601-314 from 1507328 to 1994752 tid 2 2014-07-01 17:15:19,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: writing mira0435601-314 from 1994752 to 2310144 tid 3 2014-07-01 17:15:19,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3939: read oid 151 snap -1 2014-07-01 17:15:19,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 3 to mira0435601-295 2014-07-01 17:15:19,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 4 to mira0435601-295 2014-07-01 17:15:19,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 6 to mira0435601-295 2014-07-01 17:15:19,823.823 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: finishing write tid 7 to mira0435601-295 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 295 v 370 (ObjNum 2402 snap 0 seq_num 2402) dirty exists 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 1 to mira0435601-314 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: done (3 left) 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 78 current snap is 0 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: seq_num 2404 ranges {4128768=638976,4767744=720896,5488640=81920} 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: writing mira0435601-78 from 4128768 to 4767744 tid 1 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: writing mira0435601-78 from 4767744 to 5488640 tid 2 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: writing mira0435601-78 from 5488640 to 5570560 tid 3 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3941: read oid 427 snap -1 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 2 to mira0435601-314 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 3 to mira0435601-314 2014-07-01 17:15:19,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3937: expect (ObjNum 2371 snap 0 seq_num 2371) 2014-07-01 17:15:20,770.770 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: expect (ObjNum 1221 snap 261 seq_num 1221) 2014-07-01 17:15:21,218.218 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: expect (ObjNum 1255 snap 274 seq_num 1255) 2014-07-01 17:15:21,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3736: done (5 left) 2014-07-01 17:15:21,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3737: done (4 left) 2014-07-01 17:15:21,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3738: done (3 left) 2014-07-01 17:15:21,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3739: done (2 left) 2014-07-01 17:15:21,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3740: done (1 left) 2014-07-01 17:15:21,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3741: done (0 left) 2014-07-01 17:15:21,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: read oid 285 snap -1 2014-07-01 17:15:21,929.929 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: snap_remove snap 434 2014-07-01 17:15:22,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: expect (ObjNum 1638 snap 422 seq_num 1638) 2014-07-01 17:15:22,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 5 to mira0435601-314 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 6 to mira0435601-314 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 314 v 252 (ObjNum 2403 snap 0 seq_num 2403) dirty exists 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 1 to mira0435601-78 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 2 to mira0435601-78 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 3 to mira0435601-78 2014-07-01 17:15:22,440.440 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 5 to mira0435601-78 2014-07-01 17:15:22,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 6 to mira0435601-78 2014-07-01 17:15:22,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 78 v 279 (ObjNum 2404 snap 0 seq_num 2404) dirty exists 2014-07-01 17:15:22,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3939: expect (ObjNum 2178 snap 0 seq_num 2178) 2014-07-01 17:15:22,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3742: done (1 left) 2014-07-01 17:15:22,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3743: done (0 left) 2014-07-01 17:15:22,502.502 INFO:teuthology.task.rados.rados.0.mira043.stdout:3744: rollback oid 311 current snap is 438 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 311 to 383 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: write oid 237 current snap is 438 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: seq_num 1679 ranges {565036=400000} 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: writing mira0437250-237 from 565036 to 965036 tid 1 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: read oid 281 snap -1 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: write oid 199 current snap is 438 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: seq_num 1680 ranges {508842=747424,1708841=1} 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: writing mira0437250-199 from 508842 to 1256266 tid 1 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: writing mira0437250-199 from 1708841 to 1708842 tid 2 2014-07-01 17:15:22,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3748: rollback oid 252 current snap is 438 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 252 to 430 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3746: done (4 left) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: read oid 295 snap -1 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: read oid 298 snap -1 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: delete oid 155 current snap is 438 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: finishing write tid 1 to mira0437250-237 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: expect (ObjNum 1677 snap 438 seq_num 1677) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 252 v 0 (ObjNum 32903856 snap 0 seq_num 240) dirty dne 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: finishing write tid 3 to mira0437250-237 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 311 v 1763 (ObjNum 857 snap 143 seq_num 857) dirty exists 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 1 to mira0437250-199 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 2 to mira0437250-199 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3744: done (6 left) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3748: done (5 left) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3750: done (4 left) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3751: done (3 left) 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: write oid 335 current snap is 438 2014-07-01 17:15:22,609.609 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: seq_num 1682 ranges {665254=576729,1967440=492927,2665253=1} 2014-07-01 17:15:22,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: writing mira0437250-335 from 665254 to 1241983 tid 1 2014-07-01 17:15:22,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: writing mira0437250-335 from 1967440 to 2460367 tid 2 2014-07-01 17:15:22,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: writing mira0437250-335 from 2665253 to 2665254 tid 3 2014-07-01 17:15:22,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3753: snap_create 2014-07-01 17:15:22,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 4 to mira0437250-199 2014-07-01 17:15:22,665.665 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: expect (ObjNum 626 snap 53 seq_num 626) 2014-07-01 17:15:22,723.723 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:15:22,724.724 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set unique_pool_0 pg_num 86' 2014-07-01 17:15:23,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: finishing write tid 4 to mira0437250-237 2014-07-01 17:15:23,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 237 v 1356 (ObjNum 1679 snap 438 seq_num 1679) dirty exists 2014-07-01 17:15:23,018.018 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: finishing write tid 5 to mira0437250-199 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 987 (ObjNum 1680 snap 438 seq_num 1680) dirty exists 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: finishing write tid 1 to mira0437250-335 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: finishing write tid 2 to mira0437250-335 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: finishing write tid 3 to mira0437250-335 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: finishing write tid 5 to mira0437250-335 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: finishing write tid 6 to mira0437250-335 2014-07-01 17:15:23,019.019 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 1146 (ObjNum 1682 snap 438 seq_num 1682) dirty exists 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3745: done (4 left) 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3747: done (3 left) 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3749: done (2 left) 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3752: done (1 left) 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3753: done (0 left) 2014-07-01 17:15:23,561.561 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: read oid 60 snap 416 2014-07-01 17:15:23,562.562 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: snap_remove snap 392 2014-07-01 17:15:23,577.577 INFO:teuthology.orchestra.run.mira079.stderr:set pool 3 pg_num to 86 2014-07-01 17:15:23,630.630 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: expect (ObjNum 1487 snap 365 seq_num 1487) 2014-07-01 17:15:24,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3754: done (1 left) 2014-07-01 17:15:24,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3755: done (0 left) 2014-07-01 17:15:24,694.694 INFO:teuthology.task.rados.rados.0.mira043.stdout:3756: snap_remove snap 326 2014-07-01 17:15:25,457.457 INFO:teuthology.task.rados.rados.0.mira043.stdout:3941: expect (ObjNum 1464 snap 0 seq_num 1464) 2014-07-01 17:15:25,588.588 INFO:teuthology.task.rados.rados.0.mira043.stdout:3756: done (0 left) 2014-07-01 17:15:25,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: write oid 348 current snap is 439 2014-07-01 17:15:25,589.589 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: seq_num 1683 ranges {610083=548533,1778723=714546,2986170=423913} 2014-07-01 17:15:25,608.608 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0437250-348 from 610083 to 1158616 tid 1 2014-07-01 17:15:25,633.633 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0437250-348 from 1778723 to 2493269 tid 2 2014-07-01 17:15:25,648.648 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: writing mira0437250-348 from 2986170 to 3410083 tid 3 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3758: rollback oid 349 current snap is 439 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 349 to 432 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: rollback oid 22 current snap is 439 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 22 to 422 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: write oid 357 current snap is 439 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: seq_num 1684 ranges {153888=1} 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: writing mira0437250-357 from 153888 to 153889 tid 1 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3761: snap_create 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 1 to mira0437250-348 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 2 to mira0437250-348 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 3 to mira0437250-348 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 22 v 925 (ObjNum 21 snap 0 seq_num 21) dirty exists 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 5 to mira0437250-348 2014-07-01 17:15:25,756.756 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: finishing write tid 6 to mira0437250-348 2014-07-01 17:15:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 348 v 2379 (ObjNum 1683 snap 439 seq_num 1683) dirty exists 2014-07-01 17:15:25,757.757 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 349 v 1683 (ObjNum 697 snap 90 seq_num 697) dirty exists 2014-07-01 17:15:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 1 to mira0437250-357 2014-07-01 17:15:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 3 to mira0437250-357 2014-07-01 17:15:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: finishing write tid 4 to mira0437250-357 2014-07-01 17:15:25,772.772 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 357 v 1954 (ObjNum 1684 snap 439 seq_num 1684) dirty exists 2014-07-01 17:15:26,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3757: done (4 left) 2014-07-01 17:15:26,593.593 INFO:teuthology.task.rados.rados.0.mira043.stdout:3758: done (3 left) 2014-07-01 17:15:26,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3759: done (2 left) 2014-07-01 17:15:26,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3760: done (1 left) 2014-07-01 17:15:26,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3761: done (0 left) 2014-07-01 17:15:26,594.594 INFO:teuthology.task.rados.rados.0.mira043.stdout:3762: delete oid 384 current snap is 440 2014-07-01 17:15:26,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:3762: done (0 left) 2014-07-01 17:15:26,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: read oid 176 snap -1 2014-07-01 17:15:26,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3764: snap_create 2014-07-01 17:15:26,643.643 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: expect (ObjNum 1165 snap 245 seq_num 1165) 2014-07-01 17:15:26,979.979 INFO:teuthology.task.rados.rados.0.mira043.stdout:3937: done (4 left) 2014-07-01 17:15:26,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: done (3 left) 2014-07-01 17:15:26,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:3939: done (2 left) 2014-07-01 17:15:26,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: done (1 left) 2014-07-01 17:15:26,980.980 INFO:teuthology.task.rados.rados.0.mira043.stdout:3941: done (0 left) 2014-07-01 17:15:26,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 152 current snap is 0 2014-07-01 17:15:26,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: seq_num 2405 ranges {3739648=266240} 2014-07-01 17:15:27,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: writing mira0435601-152 from 3739648 to 4005888 tid 1 2014-07-01 17:15:27,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3943: read oid 170 snap -1 2014-07-01 17:15:27,152.152 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: read oid 484 snap -1 2014-07-01 17:15:27,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 398 current snap is 0 2014-07-01 17:15:27,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: seq_num 2406 ranges {3264512=135168} 2014-07-01 17:15:27,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: writing mira0435601-398 from 3264512 to 3399680 tid 1 2014-07-01 17:15:27,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: read oid 3 snap -1 2014-07-01 17:15:27,153.153 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: finishing write tid 1 to mira0435601-152 2014-07-01 17:15:27,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: finishing write tid 3 to mira0435601-152 2014-07-01 17:15:27,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3947: delete oid 446 current snap is 0 2014-07-01 17:15:27,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: finishing write tid 1 to mira0435601-398 2014-07-01 17:15:27,154.154 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: finishing write tid 3 to mira0435601-398 2014-07-01 17:15:27,188.188 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: expect (ObjNum 2337 snap 0 seq_num 2337) 2014-07-01 17:15:27,615.615 INFO:teuthology.task.rados.rados.0.mira043.stdout:3763: done (1 left) 2014-07-01 17:15:27,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3764: done (0 left) 2014-07-01 17:15:27,616.616 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: snap_remove snap 420 2014-07-01 17:15:28,591.591 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:28,592.592 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:15:28,593.593 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:15:28,594.594 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:15:28,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3765: done (0 left) 2014-07-01 17:15:28,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: write oid 313 current snap is 441 2014-07-01 17:15:28,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: seq_num 1686 ranges {711324=400000} 2014-07-01 17:15:28,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: writing mira0437250-313 from 711324 to 1111324 tid 1 2014-07-01 17:15:28,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: write oid 397 current snap is 441 2014-07-01 17:15:28,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: seq_num 1687 ranges {655130=623270,1764083=91047} 2014-07-01 17:15:28,660.660 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: writing mira0437250-397 from 655130 to 1278400 tid 1 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: writing mira0437250-397 from 1764083 to 1855130 tid 2 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3768: read oid 149 snap 421 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3769: delete oid 105 current snap is 441 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3769: done (3 left) 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: write oid 471 current snap is 441 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: seq_num 1689 ranges {694166=602975,2087452=206714} 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0437250-471 from 694166 to 1297141 tid 1 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: writing mira0437250-471 from 2087452 to 2294166 tid 2 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: read oid 437 snap -1 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 1 to mira0437250-313 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 3 to mira0437250-313 2014-07-01 17:15:28,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: finishing write tid 1 to mira0437250-397 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: finishing write tid 2 to mira0437250-397 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: finishing write tid 4 to mira0437250-397 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: finishing write tid 5 to mira0437250-397 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 397 v 674 (ObjNum 1687 snap 441 seq_num 1687) dirty exists 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: finishing write tid 4 to mira0437250-313 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 313 v 1313 (ObjNum 1686 snap 441 seq_num 1686) dirty exists 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3768: expect (ObjNum 1560 snap 402 seq_num 1560) 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 1 to mira0437250-471 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3766: done (4 left) 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3767: done (3 left) 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3768: done (2 left) 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: rollback oid 229 current snap is 441 2014-07-01 17:15:28,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 229 to 373 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: read oid 105 snap 432 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3774: snap_create 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 2 to mira0437250-471 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 4 to mira0437250-471 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: finishing write tid 5 to mira0437250-471 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 471 v 1232 (ObjNum 1689 snap 441 seq_num 1689) dirty exists 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 229 v 1302 (ObjNum 1082 snap 217 seq_num 1082) dirty exists 2014-07-01 17:15:28,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: expect (ObjNum 1628 snap 421 seq_num 1628) 2014-07-01 17:15:29,358.358 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:15:29,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: finishing write tid 4 to mira0435601-152 2014-07-01 17:15:29,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 152 v 188 (ObjNum 2405 snap 0 seq_num 2405) dirty exists 2014-07-01 17:15:29,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: finishing write tid 4 to mira0435601-398 2014-07-01 17:15:29,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 398 v 112 (ObjNum 2406 snap 0 seq_num 2406) dirty exists 2014-07-01 17:15:29,634.634 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: expect (ObjNum 992 snap 0 seq_num 992) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3770: done (4 left) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3771: done (3 left) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3772: done (2 left) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3773: done (1 left) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3774: done (0 left) 2014-07-01 17:15:29,674.674 INFO:teuthology.task.rados.rados.0.mira043.stdout:3775: snap_create 2014-07-01 17:15:30,680.680 INFO:teuthology.task.rados.rados.0.mira043.stdout:3775: done (0 left) 2014-07-01 17:15:30,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: read oid 383 snap -1 2014-07-01 17:15:30,681.681 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: write oid 440 current snap is 443 2014-07-01 17:15:30,682.682 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: seq_num 1690 ranges {790419=726203,1953008=37411} 2014-07-01 17:15:30,722.722 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: writing mira0437250-440 from 790419 to 1516622 tid 1 2014-07-01 17:15:30,724.724 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: writing mira0437250-440 from 1953008 to 1990419 tid 2 2014-07-01 17:15:30,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: write oid 141 current snap is 443 2014-07-01 17:15:30,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: seq_num 1691 ranges {300176=1} 2014-07-01 17:15:30,843.843 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: writing mira0437250-141 from 300176 to 300177 tid 1 2014-07-01 17:15:30,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:3779: snap_create 2014-07-01 17:15:30,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: expect (ObjNum 1333 snap 308 seq_num 1333) 2014-07-01 17:15:32,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 1 to mira0437250-141 2014-07-01 17:15:32,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 3 to mira0437250-141 2014-07-01 17:15:32,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: finishing write tid 1 to mira0437250-440 2014-07-01 17:15:32,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: finishing write tid 2 to mira0437250-440 2014-07-01 17:15:32,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: finishing write tid 4 to mira0437250-440 2014-07-01 17:15:32,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: finishing write tid 5 to mira0437250-440 2014-07-01 17:15:32,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 440 v 1340 (ObjNum 1690 snap 443 seq_num 1690) dirty exists 2014-07-01 17:15:32,795.795 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: finishing write tid 4 to mira0437250-141 2014-07-01 17:15:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 141 v 1171 (ObjNum 1691 snap 443 seq_num 1691) dirty exists 2014-07-01 17:15:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:3776: done (3 left) 2014-07-01 17:15:32,796.796 INFO:teuthology.task.rados.rados.0.mira043.stdout:3777: done (2 left) 2014-07-01 17:15:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3778: done (1 left) 2014-07-01 17:15:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3779: done (0 left) 2014-07-01 17:15:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: write oid 307 current snap is 444 2014-07-01 17:15:32,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: seq_num 1692 ranges {795407=670836,1959705=732799,3489680=505727} 2014-07-01 17:15:32,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: writing mira0437250-307 from 795407 to 1466243 tid 1 2014-07-01 17:15:32,840.840 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: writing mira0437250-307 from 1959705 to 2692504 tid 2 2014-07-01 17:15:32,960.960 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: writing mira0437250-307 from 3489680 to 3995407 tid 3 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: delete oid 465 current snap is 444 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: finishing write tid 1 to mira0437250-307 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: finishing write tid 2 to mira0437250-307 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3781: done (1 left) 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3782: read oid 268 snap -1 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: write oid 429 current snap is 444 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: seq_num 1694 ranges {684042=649517,1484041=1} 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: writing mira0437250-429 from 684042 to 1333559 tid 1 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: writing mira0437250-429 from 1484041 to 1484042 tid 2 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3784: read oid 161 snap -1 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: finishing write tid 3 to mira0437250-307 2014-07-01 17:15:32,961.961 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: finishing write tid 5 to mira0437250-307 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: finishing write tid 6 to mira0437250-307 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 307 v 1790 (ObjNum 1692 snap 444 seq_num 1692) dirty exists 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3780: done (3 left) 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3782: done (2 left) 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3785: rollback oid 234 current snap is 444 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 234 to 412 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3786: delete oid 42 current snap is 444 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 234 v 1120 (ObjNum 1374 snap 312 seq_num 1374) dirty exists 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3785: done (3 left) 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3786: done (2 left) 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: snap_remove snap 429 2014-07-01 17:15:32,962.962 INFO:teuthology.task.rados.rados.0.mira043.stdout:3784: expect (ObjNum 844 snap 141 seq_num 844) 2014-07-01 17:15:33,043.043 INFO:teuthology.task.rados.rados.0.mira043.stdout:3943: expect (ObjNum 2201 snap 0 seq_num 2201) 2014-07-01 17:15:33,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 1 to mira0437250-429 2014-07-01 17:15:33,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 2 to mira0437250-429 2014-07-01 17:15:33,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 4 to mira0437250-429 2014-07-01 17:15:33,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: finishing write tid 5 to mira0437250-429 2014-07-01 17:15:33,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 429 v 1502 (ObjNum 1694 snap 444 seq_num 1694) dirty exists 2014-07-01 17:15:33,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3783: done (2 left) 2014-07-01 17:15:33,717.717 INFO:teuthology.task.rados.rados.0.mira043.stdout:3784: done (1 left) 2014-07-01 17:15:33,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3787: done (0 left) 2014-07-01 17:15:33,718.718 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: snap_remove snap 442 2014-07-01 17:15:34,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3788: done (0 left) 2014-07-01 17:15:34,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: read oid 498 snap -1 2014-07-01 17:15:34,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3790: rollback oid 411 current snap is 444 2014-07-01 17:15:34,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 411 to 383 2014-07-01 17:15:34,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: write oid 139 current snap is 444 2014-07-01 17:15:34,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: seq_num 1696 ranges {591878=745573,1391877=1} 2014-07-01 17:15:34,755.755 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: writing mira0437250-139 from 591878 to 1337451 tid 1 2014-07-01 17:15:34,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: writing mira0437250-139 from 1391877 to 1391878 tid 2 2014-07-01 17:15:34,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: write oid 419 current snap is 444 2014-07-01 17:15:34,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: seq_num 1697 ranges {784260=601026,1790398=469247,2724159=460101} 2014-07-01 17:15:34,897.897 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0437250-419 from 784260 to 1385286 tid 1 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0437250-419 from 1790398 to 2259645 tid 2 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: writing mira0437250-419 from 2724159 to 3184260 tid 3 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: read oid 485 snap -1 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: expect (ObjNum 1433 snap 341 seq_num 1433) 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 411 v 1234 (ObjNum 1430 snap 340 seq_num 1430) dirty exists 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 1 to mira0437250-139 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3789: done (4 left) 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3790: done (3 left) 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: write oid 328 current snap is 444 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: seq_num 1698 ranges {480513=724254,1904529=690866,2880512=1} 2014-07-01 17:15:34,898.898 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0437250-328 from 480513 to 1204767 tid 1 2014-07-01 17:15:34,922.922 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0437250-328 from 1904529 to 2595395 tid 2 2014-07-01 17:15:34,925.925 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: writing mira0437250-328 from 2880512 to 2880513 tid 3 2014-07-01 17:15:34,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: write oid 410 current snap is 444 2014-07-01 17:15:34,926.926 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: seq_num 1699 ranges {575743=697082,1735013=761061,2575742=1} 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: writing mira0437250-410 from 575743 to 1272825 tid 1 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: writing mira0437250-410 from 1735013 to 2496074 tid 2 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: writing mira0437250-410 from 2575742 to 2575743 tid 3 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3796: rollback oid 326 current snap is 444 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 326 to 415 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 2 to mira0437250-139 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 1 to mira0437250-419 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 2 to mira0437250-419 2014-07-01 17:15:35,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 4 to mira0437250-139 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: finishing write tid 5 to mira0437250-139 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 139 v 1102 (ObjNum 1696 snap 444 seq_num 1696) dirty exists 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 3 to mira0437250-419 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 5 to mira0437250-419 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: finishing write tid 6 to mira0437250-419 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 419 v 1324 (ObjNum 1697 snap 444 seq_num 1697) dirty exists 2014-07-01 17:15:35,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: expect (ObjNum 1623 snap 421 seq_num 1623) 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 1 to mira0437250-328 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 2 to mira0437250-328 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 3 to mira0437250-328 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 5 to mira0437250-328 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: finishing write tid 6 to mira0437250-328 2014-07-01 17:15:35,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 328 v 1679 (ObjNum 1698 snap 444 seq_num 1698) dirty exists 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 1 to mira0437250-410 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 326 v 0 (ObjNum 17415776 snap 0 seq_num 695) dirty dne 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 2 to mira0437250-410 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 3 to mira0437250-410 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 5 to mira0437250-410 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: finishing write tid 6 to mira0437250-410 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 410 v 1107 (ObjNum 1699 snap 444 seq_num 1699) dirty exists 2014-07-01 17:15:35,816.816 INFO:teuthology.task.rados.rados.0.mira043.stdout:3791: done (5 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3792: done (4 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3793: done (3 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3794: done (2 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3795: done (1 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3796: done (0 left) 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: read oid 175 snap -1 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: read oid 447 snap -1 2014-07-01 17:15:35,817.817 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: read oid 37 snap 419 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: read oid 220 snap 382 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: read oid 414 snap 438 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: delete oid 246 current snap is 444 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3798: done (5 left) 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3799: done (4 left) 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3800: done (3 left) 2014-07-01 17:15:35,981.981 INFO:teuthology.task.rados.rados.0.mira043.stdout:3801: done (2 left) 2014-07-01 17:15:35,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3802: done (1 left) 2014-07-01 17:15:35,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: read oid 71 snap -1 2014-07-01 17:15:35,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3804: delete oid 34 current snap is 444 2014-07-01 17:15:35,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: expect (ObjNum 174 snap 0 seq_num 174) 2014-07-01 17:15:36,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:3797: done (2 left) 2014-07-01 17:15:36,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:3803: done (1 left) 2014-07-01 17:15:36,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:3804: done (0 left) 2014-07-01 17:15:36,003.003 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: snap_remove snap 443 2014-07-01 17:15:36,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3805: done (0 left) 2014-07-01 17:15:36,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3806: read oid 78 snap -1 2014-07-01 17:15:36,733.733 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: read oid 398 snap -1 2014-07-01 17:15:36,734.734 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: read oid 493 snap -1 2014-07-01 17:15:36,735.735 INFO:teuthology.task.rados.rados.0.mira043.stdout:3809: delete oid 242 current snap is 444 2014-07-01 17:15:36,763.763 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: expect (ObjNum 1457 snap 353 seq_num 1457) 2014-07-01 17:15:37,041.041 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: expect (ObjNum 1369 snap 311 seq_num 1369) 2014-07-01 17:15:37,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3806: done (3 left) 2014-07-01 17:15:37,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3807: done (2 left) 2014-07-01 17:15:37,564.564 INFO:teuthology.task.rados.rados.0.mira043.stdout:3808: done (1 left) 2014-07-01 17:15:37,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3809: done (0 left) 2014-07-01 17:15:37,565.565 INFO:teuthology.task.rados.rados.0.mira043.stdout:3810: snap_create 2014-07-01 17:15:37,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:3810: done (0 left) 2014-07-01 17:15:37,754.754 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: delete oid 437 current snap is 445 2014-07-01 17:15:37,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:3811: done (0 left) 2014-07-01 17:15:37,788.788 INFO:teuthology.task.rados.rados.0.mira043.stdout:3812: snap_create 2014-07-01 17:15:37,914.914 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:37,914.914 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:15:37,914.914 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 0 2014-07-01 17:15:37,914.914 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:15:37,914.914 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:15:37,914.914 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config set filestore_inject_stall 3' 2014-07-01 17:15:38,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3812: done (0 left) 2014-07-01 17:15:38,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3813: snap_remove snap 412 2014-07-01 17:15:39,325.325 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: done (5 left) 2014-07-01 17:15:39,326.326 INFO:teuthology.task.rados.rados.0.mira043.stdout:3943: done (4 left) 2014-07-01 17:15:39,327.327 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: done (3 left) 2014-07-01 17:15:39,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: done (2 left) 2014-07-01 17:15:39,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: done (1 left) 2014-07-01 17:15:39,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3947: done (0 left) 2014-07-01 17:15:39,328.328 INFO:teuthology.task.rados.rados.0.mira043.stdout:3948: read oid 206 snap -1 2014-07-01 17:15:39,329.329 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 212 current snap is 0 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: seq_num 2408 ranges {1966080=487424,2453504=544768,2998272=569344,3567616=458752,4026368=32768} 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: writing mira0435601-212 from 1966080 to 2453504 tid 1 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: writing mira0435601-212 from 2453504 to 2998272 tid 2 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: writing mira0435601-212 from 2998272 to 3567616 tid 3 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: writing mira0435601-212 from 3567616 to 4026368 tid 4 2014-07-01 17:15:39,495.495 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: writing mira0435601-212 from 4026368 to 4059136 tid 5 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 57 current snap is 0 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: seq_num 2409 ranges {2531328=602112} 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: writing mira0435601-57 from 2531328 to 3133440 tid 1 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 315 current snap is 0 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: seq_num 2410 ranges {0=630784,630784=712704,1343488=491520} 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: writing mira0435601-315 from 0 to 630784 tid 1 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: writing mira0435601-315 from 630784 to 1343488 tid 2 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: writing mira0435601-315 from 1343488 to 1835008 tid 3 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 445 current snap is 0 2014-07-01 17:15:39,496.496 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: seq_num 2411 ranges {3678208=659456} 2014-07-01 17:15:39,525.525 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: writing mira0435601-445 from 3678208 to 4337664 tid 1 2014-07-01 17:15:39,526.526 INFO:teuthology.task.rados.rados.0.mira043.stdout:3953: read oid 301 snap -1 2014-07-01 17:15:39,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 1 to mira0435601-212 2014-07-01 17:15:39,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 2 to mira0435601-212 2014-07-01 17:15:39,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 3 to mira0435601-212 2014-07-01 17:15:39,527.527 INFO:teuthology.task.rados.rados.0.mira043.stdout:3948: expect (ObjNum 2193 snap 0 seq_num 2193) 2014-07-01 17:15:39,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3813: done (0 left) 2014-07-01 17:15:39,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: snap_remove snap 382 2014-07-01 17:15:40,785.785 INFO:teuthology.task.rados.rados.0.mira043.stdout:3814: done (0 left) 2014-07-01 17:15:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: read oid 312 snap -1 2014-07-01 17:15:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: write oid 15 current snap is 446 2014-07-01 17:15:40,786.786 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: seq_num 1704 ranges {565620=743624,1765619=1} 2014-07-01 17:15:40,809.809 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: writing mira0437250-15 from 565620 to 1309244 tid 1 2014-07-01 17:15:40,814.814 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: writing mira0437250-15 from 1765619 to 1765620 tid 2 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: write oid 274 current snap is 446 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: seq_num 1705 ranges {777201=599077,1977200=1} 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: writing mira0437250-274 from 777201 to 1376278 tid 1 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: writing mira0437250-274 from 1977200 to 1977201 tid 2 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: write oid 242 current snap is 446 2014-07-01 17:15:40,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: seq_num 1706 ranges {722031=722305,1874226=638923,2722030=1} 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0437250-242 from 722031 to 1444336 tid 1 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0437250-242 from 1874226 to 2513149 tid 2 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: writing mira0437250-242 from 2722030 to 2722031 tid 3 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3819: read oid 85 snap -1 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 1 to mira0437250-274 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 2 to mira0437250-274 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 4 to mira0437250-274 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: finishing write tid 5 to mira0437250-274 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 1176 (ObjNum 1705 snap 446 seq_num 1705) dirty exists 2014-07-01 17:15:40,955.955 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: finishing write tid 1 to mira0437250-15 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: finishing write tid 2 to mira0437250-15 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: finishing write tid 4 to mira0437250-15 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: finishing write tid 5 to mira0437250-15 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 15 v 1520 (ObjNum 1704 snap 446 seq_num 1704) dirty exists 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 1 to mira0437250-242 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3816: done (4 left) 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3817: done (3 left) 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: read oid 152 snap 411 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: write oid 168 current snap is 446 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: seq_num 1707 ranges {665837=695133,1836935=460543,2833564=489405,3465836=1} 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0437250-168 from 665837 to 1360970 tid 1 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0437250-168 from 1836935 to 2297478 tid 2 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0437250-168 from 2833564 to 3322969 tid 3 2014-07-01 17:15:40,956.956 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: writing mira0437250-168 from 3465836 to 3465837 tid 4 2014-07-01 17:15:40,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: write oid 335 current snap is 446 2014-07-01 17:15:40,957.957 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: seq_num 1708 ranges {762091=418362,1951068=530738,3162090=1} 2014-07-01 17:15:40,966.966 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: writing mira0437250-335 from 762091 to 1180453 tid 1 2014-07-01 17:15:40,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: writing mira0437250-335 from 1951068 to 2481806 tid 2 2014-07-01 17:15:40,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: writing mira0437250-335 from 3162090 to 3162091 tid 3 2014-07-01 17:15:41,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3823: rollback oid 55 current snap is 446 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 55 to 417 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: expect (ObjNum 1413 snap 328 seq_num 1413) 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 2 to mira0437250-242 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 3 to mira0437250-242 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 5 to mira0437250-242 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: finishing write tid 6 to mira0437250-242 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 242 v 1144 (ObjNum 1706 snap 446 seq_num 1706) dirty exists 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 1 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 2 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 3 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 4 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 6 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: finishing write tid 7 to mira0437250-168 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 168 v 1397 (ObjNum 1707 snap 446 seq_num 1707) dirty exists 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 55 v 1987 (ObjNum 1516 snap 378 seq_num 1516) dirty exists 2014-07-01 17:15:41,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3818: done (6 left) 2014-07-01 17:15:41,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3819: done (5 left) 2014-07-01 17:15:41,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3820: done (4 left) 2014-07-01 17:15:41,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3821: done (3 left) 2014-07-01 17:15:41,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3823: done (2 left) 2014-07-01 17:15:41,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: delete oid 305 current snap is 446 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3824: done (2 left) 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: write oid 29 current snap is 446 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: seq_num 1710 ranges {767078=400000} 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: writing mira0437250-29 from 767078 to 1167078 tid 1 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: write oid 72 current snap is 446 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: seq_num 1711 ranges {463332=400000} 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: writing mira0437250-72 from 463332 to 863332 tid 1 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: read oid 1 snap -1 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: write oid 136 current snap is 446 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: seq_num 1712 ranges {407138=494123,1536428=70710} 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: writing mira0437250-136 from 407138 to 901261 tid 1 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: writing mira0437250-136 from 1536428 to 1607138 tid 2 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: read oid 223 snap -1 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 1 to mira0437250-72 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 3 to mira0437250-72 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: write oid 111 current snap is 446 2014-07-01 17:15:41,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: seq_num 1713 ranges {502367=598151,1302366=1} 2014-07-01 17:15:41,124.124 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: writing mira0437250-111 from 502367 to 1100518 tid 1 2014-07-01 17:15:41,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: writing mira0437250-111 from 1302366 to 1302367 tid 2 2014-07-01 17:15:41,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: write oid 275 current snap is 446 2014-07-01 17:15:41,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: seq_num 1714 ranges {562525=722404,1977172=585353} 2014-07-01 17:15:41,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0437250-275 from 562525 to 1284929 tid 1 2014-07-01 17:15:41,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: writing mira0437250-275 from 1977172 to 2562525 tid 2 2014-07-01 17:15:41,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: delete oid 273 current snap is 446 2014-07-01 17:15:41,133.133 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: expect (ObjNum 0 snap 0 seq_num 0) 2014-07-01 17:15:41,296.296 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: finishing write tid 4 to mira0437250-72 2014-07-01 17:15:41,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 72 v 1990 (ObjNum 1711 snap 446 seq_num 1711) dirty exists 2014-07-01 17:15:41,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3826: done (9 left) 2014-07-01 17:15:41,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3827: done (8 left) 2014-07-01 17:15:41,358.358 INFO:teuthology.task.rados.rados.0.mira043.stdout:3832: done (7 left) 2014-07-01 17:15:41,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: write oid 458 current snap is 446 2014-07-01 17:15:41,359.359 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: seq_num 1716 ranges {603609=417436,1521788=477771,2438896=439479,3003608=1} 2014-07-01 17:15:41,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: writing mira0437250-458 from 603609 to 1021045 tid 1 2014-07-01 17:15:41,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: writing mira0437250-458 from 1521788 to 1999559 tid 2 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: writing mira0437250-458 from 2438896 to 2878375 tid 3 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: writing mira0437250-458 from 3003608 to 3003609 tid 4 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3834: snap_remove snap 439 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 1 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 2 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 3 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 4 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 6 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: finishing write tid 7 to mira0437250-458 2014-07-01 17:15:41,461.461 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 458 v 1256 (ObjNum 1716 snap 446 seq_num 1716) dirty exists 2014-07-01 17:15:41,792.792 INFO:teuthology.task.rados.rados.0.mira043.stdout:3833: done (8 left) 2014-07-01 17:15:41,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3834: done (7 left) 2014-07-01 17:15:41,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: write oid 98 current snap is 446 2014-07-01 17:15:41,793.793 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: seq_num 1717 ranges {547414=673913,2016720=699390,3305593=422398,3747413=1} 2014-07-01 17:15:41,815.815 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0437250-98 from 547414 to 1221327 tid 1 2014-07-01 17:15:41,841.841 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0437250-98 from 2016720 to 2716110 tid 2 2014-07-01 17:15:41,857.857 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0437250-98 from 3305593 to 3727991 tid 3 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: writing mira0437250-98 from 3747413 to 3747414 tid 4 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: read oid 390 snap -1 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3837: snap_remove snap 437 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 1 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 2 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 3 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 4 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 6 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: finishing write tid 7 to mira0437250-98 2014-07-01 17:15:41,959.959 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 98 v 1257 (ObjNum 1717 snap 446 seq_num 1717) dirty exists 2014-07-01 17:15:42,391.391 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 1 to mira0437250-111 2014-07-01 17:15:42,433.433 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 2 to mira0437250-111 2014-07-01 17:15:42,438.438 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 4 to mira0437250-111 2014-07-01 17:15:42,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: finishing write tid 5 to mira0437250-111 2014-07-01 17:15:42,439.439 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 111 v 1009 (ObjNum 1713 snap 446 seq_num 1713) dirty exists 2014-07-01 17:15:42,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 1 to mira0437250-136 2014-07-01 17:15:42,555.555 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 2 to mira0437250-136 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 4 to mira0437250-136 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 1 to mira0437250-335 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 2 to mira0437250-335 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 3 to mira0437250-335 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 5 to mira0437250-335 2014-07-01 17:15:42,556.556 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: expect (ObjNum 1484 snap 363 seq_num 1484) 2014-07-01 17:15:43,062.062 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: expect (ObjNum 796 snap 127 seq_num 796) 2014-07-01 17:15:43,376.376 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:43,377.377 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:15:43,377.377 INFO:teuthology.task.thrashosds.thrasher:Growing pool data 2014-07-01 17:15:43,377.377 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:15:43,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: finishing write tid 6 to mira0437250-335 2014-07-01 17:15:43,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 335 v 1167 (ObjNum 1708 snap 446 seq_num 1708) dirty exists 2014-07-01 17:15:43,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 1 to mira0437250-275 2014-07-01 17:15:43,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 2 to mira0437250-275 2014-07-01 17:15:43,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 4 to mira0437250-275 2014-07-01 17:15:43,452.452 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: finishing write tid 1 to mira0437250-29 2014-07-01 17:15:43,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: finishing write tid 3 to mira0437250-29 2014-07-01 17:15:43,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: finishing write tid 5 to mira0437250-275 2014-07-01 17:15:43,454.454 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 275 v 1820 (ObjNum 1714 snap 446 seq_num 1714) dirty exists 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: finishing write tid 4 to mira0437250-29 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 29 v 1092 (ObjNum 1710 snap 446 seq_num 1710) dirty exists 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: finishing write tid 5 to mira0437250-136 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 136 v 1626 (ObjNum 1712 snap 446 seq_num 1712) dirty exists 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3815: done (9 left) 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3822: done (8 left) 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3825: done (7 left) 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3828: done (6 left) 2014-07-01 17:15:43,455.455 INFO:teuthology.task.rados.rados.0.mira043.stdout:3829: done (5 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3830: done (4 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3831: done (3 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3835: done (2 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3836: done (1 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3837: done (0 left) 2014-07-01 17:15:43,456.456 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: write oid 266 current snap is 446 2014-07-01 17:15:43,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: seq_num 1718 ranges {608596=400000} 2014-07-01 17:15:43,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: writing mira0437250-266 from 608596 to 1008596 tid 1 2014-07-01 17:15:43,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: rollback oid 351 current snap is 446 2014-07-01 17:15:43,619.619 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 351 to 421 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: delete oid 248 current snap is 446 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 351 v 892 (ObjNum 1610 snap 415 seq_num 1610) dirty exists 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3839: done (2 left) 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3840: done (1 left) 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: write oid 317 current snap is 446 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: seq_num 1720 ranges {648656=492174,1448655=1} 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: writing mira0437250-317 from 648656 to 1140830 tid 1 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: writing mira0437250-317 from 1448655 to 1448656 tid 2 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: rollback oid 272 current snap is 446 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 272 to 410 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 1 to mira0437250-266 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 3 to mira0437250-266 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: finishing write tid 4 to mira0437250-266 2014-07-01 17:15:43,620.620 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 266 v 552 (ObjNum 1718 snap 446 seq_num 1718) dirty exists 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3838: done (2 left) 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: read oid 70 snap -1 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3844: delete oid 333 current snap is 446 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 272 v 0 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3842: done (3 left) 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3844: done (2 left) 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3845: rollback oid 422 current snap is 446 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 422 to 357 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3846: read oid 256 snap -1 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: read oid 246 snap 430 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3848: rollback oid 308 current snap is 446 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 308 to 435 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: write oid 250 current snap is 446 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: seq_num 1722 ranges {687691=471878,1714164=173527} 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: writing mira0437250-250 from 687691 to 1159569 tid 1 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: writing mira0437250-250 from 1714164 to 1887691 tid 2 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:3850: snap_remove snap 419 2014-07-01 17:15:43,622.622 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 308 v 702 (ObjNum 1083 snap 217 seq_num 1083) dirty exists 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 1 to mira0437250-250 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 2 to mira0437250-250 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 4 to mira0437250-250 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: finishing write tid 5 to mira0437250-250 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 250 v 1103 (ObjNum 1722 snap 446 seq_num 1722) dirty exists 2014-07-01 17:15:43,623.623 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: expect (ObjNum 824 snap 132 seq_num 824) 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 4 to mira0435601-212 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 5 to mira0435601-212 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 7 to mira0435601-212 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: finishing write tid 8 to mira0435601-212 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 212 v 260 (ObjNum 2408 snap 0 seq_num 2408) dirty exists 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: finishing write tid 1 to mira0435601-57 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: finishing write tid 3 to mira0435601-57 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: finishing write tid 1 to mira0435601-315 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: finishing write tid 2 to mira0435601-315 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: finishing write tid 3 to mira0435601-315 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: finishing write tid 5 to mira0435601-315 2014-07-01 17:15:43,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3948: done (5 left) 2014-07-01 17:15:43,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: done (4 left) 2014-07-01 17:15:43,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 231 current snap is 0 2014-07-01 17:15:43,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: seq_num 2412 ranges {5664768=20480} 2014-07-01 17:15:43,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: writing mira0435601-231 from 5664768 to 5685248 tid 1 2014-07-01 17:15:43,903.903 INFO:teuthology.task.rados.rados.0.mira043.stdout:3955: read oid 265 snap -1 2014-07-01 17:15:43,972.972 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:15:44,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 1 to mira0435601-445 2014-07-01 17:15:44,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 3 to mira0435601-445 2014-07-01 17:15:44,027.027 INFO:teuthology.task.rados.rados.0.mira043.stdout:3953: expect (ObjNum 2376 snap 0 seq_num 2376) 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 1 to mira0437250-317 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 2 to mira0437250-317 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 4 to mira0437250-317 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: finishing write tid 5 to mira0437250-317 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 317 v 1709 (ObjNum 1720 snap 446 seq_num 1720) dirty exists 2014-07-01 17:15:44,697.697 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: expect (ObjNum 69 snap 0 seq_num 69) 2014-07-01 17:15:45,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 422 v 1486 (ObjNum 1362 snap 311 seq_num 1362) dirty exists 2014-07-01 17:15:45,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:3841: done (7 left) 2014-07-01 17:15:45,093.093 INFO:teuthology.task.rados.rados.0.mira043.stdout:3843: done (6 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3845: done (5 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3846: done (4 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3847: done (3 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3848: done (2 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3849: done (1 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3850: done (0 left) 2014-07-01 17:15:45,094.094 INFO:teuthology.task.rados.rados.0.mira043.stdout:3851: snap_create 2014-07-01 17:15:45,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3851: done (0 left) 2014-07-01 17:15:45,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: read oid 277 snap -1 2014-07-01 17:15:45,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: write oid 342 current snap is 447 2014-07-01 17:15:45,873.873 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: seq_num 1723 ranges {748873=443683,1641801=506034,2608050=483749,3148872=1} 2014-07-01 17:15:45,876.876 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: writing mira0437250-342 from 748873 to 1192556 tid 1 2014-07-01 17:15:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: writing mira0437250-342 from 1641801 to 2147835 tid 2 2014-07-01 17:15:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: writing mira0437250-342 from 2608050 to 3091799 tid 3 2014-07-01 17:15:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: writing mira0437250-342 from 3148872 to 3148873 tid 4 2014-07-01 17:15:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3854: read oid 181 snap 411 2014-07-01 17:15:46,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: expect (ObjNum 1360 snap 309 seq_num 1360) 2014-07-01 17:15:46,374.374 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 1 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 2 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 3 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 4 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 6 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: finishing write tid 7 to mira0437250-342 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 342 v 1166 (ObjNum 1723 snap 447 seq_num 1723) dirty exists 2014-07-01 17:15:46,375.375 INFO:teuthology.task.rados.rados.0.mira043.stdout:3852: done (2 left) 2014-07-01 17:15:46,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3853: done (1 left) 2014-07-01 17:15:46,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3854: done (0 left) 2014-07-01 17:15:46,376.376 INFO:teuthology.task.rados.rados.0.mira043.stdout:3855: snap_remove snap 436 2014-07-01 17:15:46,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3855: done (0 left) 2014-07-01 17:15:46,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: snap_remove snap 431 2014-07-01 17:15:47,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: finishing write tid 4 to mira0435601-57 2014-07-01 17:15:47,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 57 v 175 (ObjNum 2409 snap 0 seq_num 2409) dirty exists 2014-07-01 17:15:47,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: finishing write tid 6 to mira0435601-315 2014-07-01 17:15:47,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 315 v 355 (ObjNum 2410 snap 0 seq_num 2410) dirty exists 2014-07-01 17:15:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 4 to mira0435601-445 2014-07-01 17:15:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 445 v 254 (ObjNum 2411 snap 0 seq_num 2411) dirty exists 2014-07-01 17:15:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: finishing write tid 1 to mira0435601-231 2014-07-01 17:15:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: finishing write tid 3 to mira0435601-231 2014-07-01 17:15:47,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: finishing write tid 4 to mira0435601-231 2014-07-01 17:15:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 231 v 189 (ObjNum 2412 snap 0 seq_num 2412) dirty exists 2014-07-01 17:15:47,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3955: expect (ObjNum 2131 snap 0 seq_num 2131) 2014-07-01 17:15:47,723.723 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:15:47,724.724 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set data pg_num 44' 2014-07-01 17:15:47,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3856: done (0 left) 2014-07-01 17:15:47,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: rollback oid 285 current snap is 447 2014-07-01 17:15:47,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 285 to 417 2014-07-01 17:15:47,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: read oid 266 snap -1 2014-07-01 17:15:47,880.880 INFO:teuthology.task.rados.rados.0.mira043.stdout:3859: read oid 79 snap -1 2014-07-01 17:15:47,881.881 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: snap_remove snap 424 2014-07-01 17:15:48,045.045 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: expect (ObjNum 1718 snap 446 seq_num 1718) 2014-07-01 17:15:48,194.194 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 285 v 1660 (ObjNum 1399 snap 319 seq_num 1399) dirty exists 2014-07-01 17:15:48,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3857: done (3 left) 2014-07-01 17:15:48,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3858: done (2 left) 2014-07-01 17:15:48,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3859: done (1 left) 2014-07-01 17:15:48,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3860: done (0 left) 2014-07-01 17:15:48,886.886 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: snap_create 2014-07-01 17:15:48,914.914 INFO:teuthology.orchestra.run.mira079.stderr:set pool 0 pg_num to 44 2014-07-01 17:15:49,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3861: done (0 left) 2014-07-01 17:15:49,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: write oid 443 current snap is 448 2014-07-01 17:15:49,896.896 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: seq_num 1724 ranges {692678=416511,1720861=576229,3022779=769515,3892677=1} 2014-07-01 17:15:49,911.911 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0437250-443 from 692678 to 1109189 tid 1 2014-07-01 17:15:49,932.932 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0437250-443 from 1720861 to 2297090 tid 2 2014-07-01 17:15:50,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0437250-443 from 3022779 to 3792294 tid 3 2014-07-01 17:15:50,060.060 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: writing mira0437250-443 from 3892677 to 3892678 tid 4 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3863: snap_remove snap 440 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 1 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 2 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 3 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 4 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 6 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: finishing write tid 7 to mira0437250-443 2014-07-01 17:15:50,061.061 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 443 v 1363 (ObjNum 1724 snap 448 seq_num 1724) dirty exists 2014-07-01 17:15:50,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: done (5 left) 2014-07-01 17:15:50,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: done (4 left) 2014-07-01 17:15:50,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: done (3 left) 2014-07-01 17:15:50,270.270 INFO:teuthology.task.rados.rados.0.mira043.stdout:3953: done (2 left) 2014-07-01 17:15:50,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: done (1 left) 2014-07-01 17:15:50,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:3955: done (0 left) 2014-07-01 17:15:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 199 current snap is 0 2014-07-01 17:15:50,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: seq_num 2413 ranges {1634304=450560,2084864=512000,2596864=290816} 2014-07-01 17:15:50,297.297 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: writing mira0435601-199 from 1634304 to 2084864 tid 1 2014-07-01 17:15:50,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: writing mira0435601-199 from 2084864 to 2596864 tid 2 2014-07-01 17:15:50,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: writing mira0435601-199 from 2596864 to 2887680 tid 3 2014-07-01 17:15:50,441.441 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 190 current snap is 0 2014-07-01 17:15:50,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: seq_num 2414 ranges {2707456=77824} 2014-07-01 17:15:50,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: writing mira0435601-190 from 2707456 to 2785280 tid 1 2014-07-01 17:15:50,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 327 current snap is 0 2014-07-01 17:15:50,442.442 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: seq_num 2415 ranges {6516736=593920,7110656=401408} 2014-07-01 17:15:50,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: writing mira0435601-327 from 6516736 to 7110656 tid 1 2014-07-01 17:15:50,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: writing mira0435601-327 from 7110656 to 7512064 tid 2 2014-07-01 17:15:50,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3959: read oid 262 snap -1 2014-07-01 17:15:50,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: finishing write tid 1 to mira0435601-199 2014-07-01 17:15:50,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: finishing write tid 2 to mira0435601-199 2014-07-01 17:15:50,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: finishing write tid 3 to mira0435601-199 2014-07-01 17:15:50,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: finishing write tid 5 to mira0435601-199 2014-07-01 17:15:50,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: finishing write tid 6 to mira0435601-199 2014-07-01 17:15:50,444.444 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 145 (ObjNum 2413 snap 0 seq_num 2413) dirty exists 2014-07-01 17:15:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: finishing write tid 1 to mira0435601-190 2014-07-01 17:15:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: finishing write tid 3 to mira0435601-190 2014-07-01 17:15:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: finishing write tid 4 to mira0435601-190 2014-07-01 17:15:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 190 v 295 (ObjNum 2414 snap 0 seq_num 2414) dirty exists 2014-07-01 17:15:50,445.445 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: done (3 left) 2014-07-01 17:15:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: done (2 left) 2014-07-01 17:15:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 260 current snap is 0 2014-07-01 17:15:50,446.446 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: seq_num 2416 ranges {3534848=622592,4157440=753664,4911104=458752,5369856=393216} 2014-07-01 17:15:50,451.451 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: writing mira0435601-260 from 3534848 to 4157440 tid 1 2014-07-01 17:15:50,469.469 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: writing mira0435601-260 from 4157440 to 4911104 tid 2 2014-07-01 17:15:50,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: writing mira0435601-260 from 4911104 to 5369856 tid 3 2014-07-01 17:15:50,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: writing mira0435601-260 from 5369856 to 5763072 tid 4 2014-07-01 17:15:50,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 1 to mira0435601-327 2014-07-01 17:15:50,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 2 to mira0435601-327 2014-07-01 17:15:50,604.604 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 4 to mira0435601-327 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 5 to mira0435601-327 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 327 v 286 (ObjNum 2415 snap 0 seq_num 2415) dirty exists 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 1 to mira0435601-260 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 2 to mira0435601-260 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 3 to mira0435601-260 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3961: read oid 22 snap -1 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: done (3 left) 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3959: done (2 left) 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 495 current snap is 0 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: seq_num 2417 ranges {0=696320,696320=737280,1433600=466944} 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: writing mira0435601-495 from 0 to 696320 tid 1 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: writing mira0435601-495 from 696320 to 1433600 tid 2 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: writing mira0435601-495 from 1433600 to 1900544 tid 3 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 11 current snap is 0 2014-07-01 17:15:50,605.605 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: seq_num 2418 ranges {9986048=569344,10555392=516096,11071488=405504,11476992=282624} 2014-07-01 17:15:50,624.624 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: writing mira0435601-11 from 9986048 to 10555392 tid 1 2014-07-01 17:15:50,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: writing mira0435601-11 from 10555392 to 11071488 tid 2 2014-07-01 17:15:50,650.650 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: writing mira0435601-11 from 11071488 to 11476992 tid 3 2014-07-01 17:15:50,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: writing mira0435601-11 from 11476992 to 11759616 tid 4 2014-07-01 17:15:50,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: read oid 255 snap -1 2014-07-01 17:15:50,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 4 to mira0435601-260 2014-07-01 17:15:50,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 6 to mira0435601-260 2014-07-01 17:15:50,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3961: expect (ObjNum 1670 snap 0 seq_num 1670) 2014-07-01 17:15:50,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:3862: done (1 left) 2014-07-01 17:15:50,909.909 INFO:teuthology.task.rados.rados.0.mira043.stdout:3863: done (0 left) 2014-07-01 17:15:50,910.910 INFO:teuthology.task.rados.rados.0.mira043.stdout:3864: snap_create 2014-07-01 17:15:51,917.917 INFO:teuthology.task.rados.rados.0.mira043.stdout:3864: done (0 left) 2014-07-01 17:15:51,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: write oid 373 current snap is 449 2014-07-01 17:15:51,918.918 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: seq_num 1725 ranges {788932=788315,2234993=797848,3588931=1} 2014-07-01 17:15:51,945.945 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: writing mira0437250-373 from 788932 to 1577247 tid 1 2014-07-01 17:15:51,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: writing mira0437250-373 from 2234993 to 3032841 tid 2 2014-07-01 17:15:51,978.978 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: writing mira0437250-373 from 3588931 to 3588932 tid 3 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3866: delete oid 398 current snap is 449 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: finishing write tid 1 to mira0437250-373 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3866: done (1 left) 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3867: snap_create 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: finishing write tid 2 to mira0437250-373 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: finishing write tid 3 to mira0437250-373 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: finishing write tid 5 to mira0437250-373 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: finishing write tid 6 to mira0437250-373 2014-07-01 17:15:52,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 373 v 1687 (ObjNum 1725 snap 449 seq_num 1725) dirty exists 2014-07-01 17:15:52,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3865: done (1 left) 2014-07-01 17:15:52,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3867: done (0 left) 2014-07-01 17:15:52,928.928 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: snap_create 2014-07-01 17:15:53,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: finishing write tid 7 to mira0435601-260 2014-07-01 17:15:53,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 271 (ObjNum 2416 snap 0 seq_num 2416) dirty exists 2014-07-01 17:15:53,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: finishing write tid 1 to mira0435601-495 2014-07-01 17:15:53,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: finishing write tid 2 to mira0435601-495 2014-07-01 17:15:53,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: done (4 left) 2014-07-01 17:15:53,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3961: done (3 left) 2014-07-01 17:15:53,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 243 current snap is 0 2014-07-01 17:15:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: seq_num 2419 ranges {4407296=442368,4849664=503808,5353472=634880,5988352=65536} 2014-07-01 17:15:53,131.131 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: writing mira0435601-243 from 4407296 to 4849664 tid 1 2014-07-01 17:15:53,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: writing mira0435601-243 from 4849664 to 5353472 tid 2 2014-07-01 17:15:53,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: writing mira0435601-243 from 5353472 to 5988352 tid 3 2014-07-01 17:15:53,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: writing mira0435601-243 from 5988352 to 6053888 tid 4 2014-07-01 17:15:53,237.237 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 330 current snap is 0 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: seq_num 2420 ranges {2093056=466944} 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: writing mira0435601-330 from 2093056 to 2560000 tid 1 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 247 current snap is 0 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: seq_num 2421 ranges {1122304=790528,1912832=401408} 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: writing mira0435601-247 from 1122304 to 1912832 tid 1 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: writing mira0435601-247 from 1912832 to 2314240 tid 2 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3968: read oid 238 snap -1 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: finishing write tid 3 to mira0435601-495 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: finishing write tid 5 to mira0435601-495 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: finishing write tid 6 to mira0435601-495 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 495 v 251 (ObjNum 2417 snap 0 seq_num 2417) dirty exists 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 1 to mira0435601-11 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: done (6 left) 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 391 current snap is 0 2014-07-01 17:15:53,238.238 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: seq_num 2422 ranges {10649600=663552,11313152=401408} 2014-07-01 17:15:53,305.305 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: writing mira0435601-391 from 10649600 to 11313152 tid 1 2014-07-01 17:15:53,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: writing mira0435601-391 from 11313152 to 11714560 tid 2 2014-07-01 17:15:53,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3970: delete oid 328 current snap is 0 2014-07-01 17:15:53,318.318 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 2 to mira0435601-11 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 3 to mira0435601-11 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 4 to mira0435601-11 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 6 to mira0435601-11 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: finishing write tid 7 to mira0435601-11 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 11 v 225 (ObjNum 2418 snap 0 seq_num 2418) dirty exists 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 1 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 2 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 3 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 4 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 6 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: finishing write tid 7 to mira0435601-243 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 243 v 285 (ObjNum 2419 snap 0 seq_num 2419) dirty exists 2014-07-01 17:15:53,404.404 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 1 to mira0435601-330 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 3 to mira0435601-330 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 4 to mira0435601-330 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 330 v 362 (ObjNum 2420 snap 0 seq_num 2420) dirty exists 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: finishing write tid 1 to mira0435601-247 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: finishing write tid 2 to mira0435601-247 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: finishing write tid 4 to mira0435601-247 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: finishing write tid 5 to mira0435601-247 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 247 v 339 (ObjNum 2421 snap 0 seq_num 2421) dirty exists 2014-07-01 17:15:53,405.405 INFO:teuthology.task.rados.rados.0.mira043.stdout:3968: expect (ObjNum 2297 snap 0 seq_num 2297) 2014-07-01 17:15:53,925.925 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 5, 1] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:15:53,925.925 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 4 1' 2014-07-01 17:15:53,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3868: done (0 left) 2014-07-01 17:15:53,927.927 INFO:teuthology.task.rados.rados.0.mira043.stdout:3869: snap_remove snap 448 2014-07-01 17:15:54,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:3869: done (0 left) 2014-07-01 17:15:54,950.950 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: read oid 75 snap -1 2014-07-01 17:15:54,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: read oid 189 snap -1 2014-07-01 17:15:54,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: read oid 379 snap -1 2014-07-01 17:15:54,951.951 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: read oid 96 snap 445 2014-07-01 17:15:54,952.952 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.4 to 1 (8655362) 2014-07-01 17:15:54,973.973 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 0 1' 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: write oid 260 current snap is 451 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: seq_num 1727 ranges {793920=766996,1593919=1} 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: writing mira0437250-260 from 793920 to 1560916 tid 1 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: writing mira0437250-260 from 1593919 to 1593920 tid 2 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: delete oid 406 current snap is 451 2014-07-01 17:15:55,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: expect (ObjNum 1576 snap 405 seq_num 1576) 2014-07-01 17:15:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: finishing write tid 1 to mira0435601-391 2014-07-01 17:15:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: finishing write tid 2 to mira0435601-391 2014-07-01 17:15:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: finishing write tid 4 to mira0435601-391 2014-07-01 17:15:55,169.169 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: finishing write tid 5 to mira0435601-391 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 391 v 358 (ObjNum 2422 snap 0 seq_num 2422) dirty exists 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: done (7 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: done (6 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: done (5 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: done (4 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: done (3 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3968: done (2 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: done (1 left) 2014-07-01 17:15:55,171.171 INFO:teuthology.task.rados.rados.0.mira043.stdout:3970: done (0 left) 2014-07-01 17:15:55,172.172 INFO:teuthology.task.rados.rados.0.mira043.stdout:3971: read oid 63 snap -1 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3972: read oid 21 snap -1 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: read oid 476 snap -1 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 492 current snap is 0 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: seq_num 2424 ranges {8810496=561152,9371648=712704,10084352=753664,10838016=139264} 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: writing mira0435601-492 from 8810496 to 9371648 tid 1 2014-07-01 17:15:55,279.279 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: writing mira0435601-492 from 9371648 to 10084352 tid 2 2014-07-01 17:15:55,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: writing mira0435601-492 from 10084352 to 10838016 tid 3 2014-07-01 17:15:55,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: writing mira0435601-492 from 10838016 to 10977280 tid 4 2014-07-01 17:15:55,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 17:15:55,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: seq_num 2425 ranges {3915776=434176,4349952=696320,5046272=630784,5677056=278528} 2014-07-01 17:15:55,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: writing mira0435601-273 from 3915776 to 4349952 tid 1 2014-07-01 17:15:55,351.351 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: writing mira0435601-273 from 4349952 to 5046272 tid 2 2014-07-01 17:15:55,369.369 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: writing mira0435601-273 from 5046272 to 5677056 tid 3 2014-07-01 17:15:55,378.378 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: writing mira0435601-273 from 5677056 to 5955584 tid 4 2014-07-01 17:15:55,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 199 current snap is 0 2014-07-01 17:15:55,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: seq_num 2426 ranges {2887680=507904,3395584=479232,3874816=700416,4575232=24576} 2014-07-01 17:15:55,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: writing mira0435601-199 from 2887680 to 3395584 tid 1 2014-07-01 17:15:55,443.443 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: writing mira0435601-199 from 3395584 to 3874816 tid 2 2014-07-01 17:15:55,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: writing mira0435601-199 from 3874816 to 4575232 tid 3 2014-07-01 17:15:55,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: writing mira0435601-199 from 4575232 to 4599808 tid 4 2014-07-01 17:15:55,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: delete oid 295 current snap is 0 2014-07-01 17:15:55,501.501 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: expect (ObjNum 2218 snap 0 seq_num 2218) 2014-07-01 17:15:55,635.635 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: expect (ObjNum 1571 snap 403 seq_num 1571) 2014-07-01 17:15:55,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 1 to mira0435601-492 2014-07-01 17:15:55,707.707 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 2 to mira0435601-492 2014-07-01 17:15:55,708.708 INFO:teuthology.task.rados.rados.0.mira043.stdout:3972: expect (ObjNum 2042 snap 0 seq_num 2042) 2014-07-01 17:15:55,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: finishing write tid 1 to mira0437250-260 2014-07-01 17:15:55,844.844 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: expect (ObjNum 1519 snap 378 seq_num 1519) 2014-07-01 17:15:55,973.973 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.0 to 1 (8655362) 2014-07-01 17:15:55,987.987 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 2 1' 2014-07-01 17:15:56,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: finishing write tid 2 to mira0437250-260 2014-07-01 17:15:56,877.877 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: finishing write tid 4 to mira0437250-260 2014-07-01 17:15:56,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: finishing write tid 5 to mira0437250-260 2014-07-01 17:15:56,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 260 v 1150 (ObjNum 1727 snap 451 seq_num 1727) dirty exists 2014-07-01 17:15:56,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3870: done (5 left) 2014-07-01 17:15:56,878.878 INFO:teuthology.task.rados.rados.0.mira043.stdout:3871: done (4 left) 2014-07-01 17:15:56,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3872: done (3 left) 2014-07-01 17:15:56,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3873: done (2 left) 2014-07-01 17:15:56,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3874: done (1 left) 2014-07-01 17:15:56,879.879 INFO:teuthology.task.rados.rados.0.mira043.stdout:3875: done (0 left) 2014-07-01 17:15:56,992.992 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.2 to 1 (8655362) 2014-07-01 17:15:57,007.007 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 3 1' 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: rollback oid 462 current snap is 451 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 462 to 432 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3877: delete oid 387 current snap is 451 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 462 v 1354 (ObjNum 461 snap 0 seq_num 461) dirty exists 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3876: done (1 left) 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3877: done (0 left) 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: read oid 76 snap -1 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: snap_create 2014-07-01 17:15:57,040.040 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: expect (ObjNum 1352 snap 309 seq_num 1352) 2014-07-01 17:15:57,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3878: done (1 left) 2014-07-01 17:15:57,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3879: done (0 left) 2014-07-01 17:15:57,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: write oid 291 current snap is 452 2014-07-01 17:15:57,291.291 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: seq_num 1730 ranges {777785=469929,1664662=783896,2777784=1} 2014-07-01 17:15:57,313.313 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: writing mira0437250-291 from 777785 to 1247714 tid 1 2014-07-01 17:15:57,336.336 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: writing mira0437250-291 from 1664662 to 2448558 tid 2 2014-07-01 17:15:57,339.339 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: writing mira0437250-291 from 2777784 to 2777785 tid 3 2014-07-01 17:15:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: write oid 388 current snap is 452 2014-07-01 17:15:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: seq_num 1731 ranges {474039=442757,1496170=605515,2474038=1} 2014-07-01 17:15:57,458.458 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: writing mira0437250-388 from 474039 to 916796 tid 1 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: writing mira0437250-388 from 1496170 to 2101685 tid 2 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: writing mira0437250-388 from 2474038 to 2474039 tid 3 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: write oid 367 current snap is 452 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: seq_num 1732 ranges {534196=565986,1574206=675710,3013382=584224,3734195=1} 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: writing mira0437250-367 from 534196 to 1100182 tid 1 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: writing mira0437250-367 from 1574206 to 2249916 tid 2 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: writing mira0437250-367 from 3013382 to 3597606 tid 3 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: writing mira0437250-367 from 3734195 to 3734196 tid 4 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3883: rollback oid 227 current snap is 452 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 227 to 450 2014-07-01 17:15:57,459.459 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: finishing write tid 1 to mira0437250-291 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: finishing write tid 2 to mira0437250-291 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: finishing write tid 3 to mira0437250-291 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: finishing write tid 5 to mira0437250-291 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: finishing write tid 6 to mira0437250-291 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 291 v 1732 (ObjNum 1730 snap 452 seq_num 1730) dirty exists 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: finishing write tid 1 to mira0437250-388 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: finishing write tid 2 to mira0437250-388 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: finishing write tid 3 to mira0437250-388 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: finishing write tid 5 to mira0437250-388 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: finishing write tid 6 to mira0437250-388 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 388 v 893 (ObjNum 1731 snap 452 seq_num 1731) dirty exists 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 1 to mira0437250-367 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3880: done (3 left) 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3881: done (2 left) 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: snap_create 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 2 to mira0437250-367 2014-07-01 17:15:57,460.460 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 227 v 1620 (ObjNum 1359 snap 309 seq_num 1359) dirty exists 2014-07-01 17:15:57,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 3 to mira0437250-367 2014-07-01 17:15:57,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 4 to mira0437250-367 2014-07-01 17:15:57,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 6 to mira0437250-367 2014-07-01 17:15:57,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: finishing write tid 7 to mira0437250-367 2014-07-01 17:15:57,476.476 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 367 v 1670 (ObjNum 1732 snap 452 seq_num 1732) dirty exists 2014-07-01 17:15:57,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3882: done (2 left) 2014-07-01 17:15:57,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3883: done (1 left) 2014-07-01 17:15:57,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3884: done (0 left) 2014-07-01 17:15:57,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3885: snap_create 2014-07-01 17:15:58,013.013 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.3 to 1 (8655362) 2014-07-01 17:15:58,029.029 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 5 1' 2014-07-01 17:15:58,982.982 INFO:teuthology.task.rados.rados.0.mira043.stdout:3885: done (0 left) 2014-07-01 17:15:58,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: write oid 26 current snap is 454 2014-07-01 17:15:58,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: seq_num 1733 ranges {479025=1} 2014-07-01 17:15:58,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: writing mira0437250-26 from 479025 to 479026 tid 1 2014-07-01 17:15:58,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: write oid 83 current snap is 454 2014-07-01 17:15:58,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: seq_num 1734 ranges {174255=1} 2014-07-01 17:15:59,005.005 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.5 to 1 (8655362) 2014-07-01 17:15:59,016.016 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd reweight 1 1' 2014-07-01 17:15:59,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: writing mira0437250-83 from 174255 to 174256 tid 1 2014-07-01 17:15:59,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: read oid 122 snap 432 2014-07-01 17:15:59,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: rollback oid 437 current snap is 454 2014-07-01 17:15:59,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 437 to 357 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: snap_remove snap 361 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: finishing write tid 1 to mira0437250-83 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: finishing write tid 3 to mira0437250-83 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: finishing write tid 4 to mira0437250-83 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 83 v 865 (ObjNum 1734 snap 454 seq_num 1734) dirty exists 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 437 v 1951 (ObjNum 816 snap 130 seq_num 816) dirty exists 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 1 to mira0437250-26 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 3 to mira0437250-26 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: finishing write tid 4 to mira0437250-26 2014-07-01 17:15:59,149.149 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 26 v 1260 (ObjNum 1733 snap 454 seq_num 1733) dirty exists 2014-07-01 17:16:00,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3886: done (4 left) 2014-07-01 17:16:00,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3887: done (3 left) 2014-07-01 17:16:00,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3888: done (2 left) 2014-07-01 17:16:00,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3889: done (1 left) 2014-07-01 17:16:00,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3890: done (0 left) 2014-07-01 17:16:00,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:3891: rollback oid 464 current snap is 454 2014-07-01 17:16:00,038.038 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 464 to 427 2014-07-01 17:16:00,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:3892: read oid 307 snap -1 2014-07-01 17:16:00,039.039 INFO:teuthology.task.rados.rados.0.mira043.stdout:3893: snap_create 2014-07-01 17:16:00,076.076 INFO:teuthology.orchestra.run.mira079.stderr:reweighted osd.1 to 1 (8655362) 2014-07-01 17:16:00,083.083 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 1345 (ObjNum 1398 snap 319 seq_num 1398) dirty exists 2014-07-01 17:16:00,083.083 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 17:16:00,083.083 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:16:00,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3892: expect (ObjNum 1692 snap 444 seq_num 1692) 2014-07-01 17:16:01,011.011 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:16:01,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 3 to mira0435601-492 2014-07-01 17:16:01,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 4 to mira0435601-492 2014-07-01 17:16:01,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 6 to mira0435601-492 2014-07-01 17:16:01,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: finishing write tid 7 to mira0435601-492 2014-07-01 17:16:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 238 (ObjNum 2424 snap 0 seq_num 2424) dirty exists 2014-07-01 17:16:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 1 to mira0435601-273 2014-07-01 17:16:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 2 to mira0435601-273 2014-07-01 17:16:01,121.121 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 3 to mira0435601-273 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 4 to mira0435601-273 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 6 to mira0435601-273 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 7 to mira0435601-273 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 203 (ObjNum 2425 snap 0 seq_num 2425) dirty exists 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 1 to mira0435601-199 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 2 to mira0435601-199 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 3 to mira0435601-199 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 4 to mira0435601-199 2014-07-01 17:16:01,122.122 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 6 to mira0435601-199 2014-07-01 17:16:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: finishing write tid 7 to mira0435601-199 2014-07-01 17:16:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 199 v 150 (ObjNum 2426 snap 0 seq_num 2426) dirty exists 2014-07-01 17:16:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3971: done (6 left) 2014-07-01 17:16:01,123.123 INFO:teuthology.task.rados.rados.0.mira043.stdout:3972: done (5 left) 2014-07-01 17:16:01,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: done (4 left) 2014-07-01 17:16:01,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: done (3 left) 2014-07-01 17:16:01,289.289 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: done (2 left) 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: done (1 left) 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: done (0 left) 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3978: read oid 82 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: read oid 422 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3980: read oid 209 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: read oid 218 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3982: read oid 415 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 51 current snap is 0 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: seq_num 2428 ranges {1949696=655360,2605056=536576,3141632=266240} 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: writing mira0435601-51 from 1949696 to 2605056 tid 1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: writing mira0435601-51 from 2605056 to 3141632 tid 2 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: writing mira0435601-51 from 3141632 to 3407872 tid 3 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 273 current snap is 0 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: seq_num 2429 ranges {5955584=278528} 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: writing mira0435601-273 from 5955584 to 6234112 tid 1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: read oid 167 snap -1 2014-07-01 17:16:01,290.290 INFO:teuthology.task.rados.rados.0.mira043.stdout:3978: expect (ObjNum 2017 snap 0 seq_num 2017) 2014-07-01 17:16:01,745.745 INFO:teuthology.task.rados.rados.0.mira043.stdout:3982: expect (ObjNum 2379 snap 0 seq_num 2379) 2014-07-01 17:16:01,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:3891: done (2 left) 2014-07-01 17:16:01,915.915 INFO:teuthology.task.rados.rados.0.mira043.stdout:3892: done (1 left) 2014-07-01 17:16:01,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:3893: done (0 left) 2014-07-01 17:16:01,916.916 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: delete oid 96 current snap is 455 2014-07-01 17:16:01,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3894: done (0 left) 2014-07-01 17:16:01,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: rollback oid 50 current snap is 455 2014-07-01 17:16:01,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 50 to 417 2014-07-01 17:16:01,923.923 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: write oid 481 current snap is 455 2014-07-01 17:16:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: seq_num 1736 ranges {580267=489299,1808142=372125} 2014-07-01 17:16:01,947.947 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: writing mira0437250-481 from 580267 to 1069566 tid 1 2014-07-01 17:16:02,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: writing mira0437250-481 from 1808142 to 2180267 tid 2 2014-07-01 17:16:02,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: read oid 388 snap -1 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 50 v 1943 (ObjNum 49 snap 0 seq_num 49) dirty exists 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: finishing write tid 1 to mira0437250-481 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3895: done (2 left) 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3898: snap_create 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: finishing write tid 2 to mira0437250-481 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: finishing write tid 4 to mira0437250-481 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: finishing write tid 5 to mira0437250-481 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 481 v 1432 (ObjNum 1736 snap 455 seq_num 1736) dirty exists 2014-07-01 17:16:02,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: expect (ObjNum 1731 snap 452 seq_num 1731) 2014-07-01 17:16:03,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3896: done (2 left) 2014-07-01 17:16:03,069.069 INFO:teuthology.task.rados.rados.0.mira043.stdout:3897: done (1 left) 2014-07-01 17:16:03,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3898: done (0 left) 2014-07-01 17:16:03,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: write oid 346 current snap is 456 2014-07-01 17:16:03,070.070 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: seq_num 1737 ranges {674473=461104,1874472=1} 2014-07-01 17:16:03,085.085 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: writing mira0437250-346 from 674473 to 1135577 tid 1 2014-07-01 17:16:03,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: writing mira0437250-346 from 1874472 to 1874473 tid 2 2014-07-01 17:16:03,092.092 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: read oid 486 snap -1 2014-07-01 17:16:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: snap_create 2014-07-01 17:16:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: finishing write tid 1 to mira0437250-346 2014-07-01 17:16:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: finishing write tid 2 to mira0437250-346 2014-07-01 17:16:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: finishing write tid 4 to mira0437250-346 2014-07-01 17:16:03,233.233 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: expect (ObjNum 1188 snap 251 seq_num 1188) 2014-07-01 17:16:03,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: finishing write tid 5 to mira0437250-346 2014-07-01 17:16:03,587.587 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 1368 (ObjNum 1737 snap 456 seq_num 1737) dirty exists 2014-07-01 17:16:04,048.048 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: expect (ObjNum 2060 snap 0 seq_num 2060) 2014-07-01 17:16:04,049.049 INFO:teuthology.task.rados.rados.0.mira043.stdout:3899: done (2 left) 2014-07-01 17:16:04,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:3900: done (1 left) 2014-07-01 17:16:04,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:3901: done (0 left) 2014-07-01 17:16:04,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:3902: rollback oid 331 current snap is 457 2014-07-01 17:16:04,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 331 to 415 2014-07-01 17:16:04,051.051 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: snap_create 2014-07-01 17:16:04,082.082 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 331 v 1208 (ObjNum 1307 snap 297 seq_num 1307) dirty exists 2014-07-01 17:16:04,572.572 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:16:05,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:3902: done (1 left) 2014-07-01 17:16:05,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:3903: done (0 left) 2014-07-01 17:16:05,063.063 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: snap_create 2014-07-01 17:16:05,269.269 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:16:05,396.396 INFO:teuthology.task.rados.rados.0.mira043.stdout:3980: expect (ObjNum 2396 snap 0 seq_num 2396) 2014-07-01 17:16:06,064.064 INFO:teuthology.task.rados.rados.0.mira043.stdout:3904: done (0 left) 2014-07-01 17:16:06,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: write oid 5 current snap is 459 2014-07-01 17:16:06,065.065 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: seq_num 1738 ranges {619303=469004,1767607=482353,2619302=1} 2014-07-01 17:16:06,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: writing mira0437250-5 from 619303 to 1088307 tid 1 2014-07-01 17:16:06,120.120 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: writing mira0437250-5 from 1767607 to 2249960 tid 2 2014-07-01 17:16:06,125.125 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: writing mira0437250-5 from 2619302 to 2619303 tid 3 2014-07-01 17:16:06,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:3906: delete oid 226 current snap is 459 2014-07-01 17:16:06,229.229 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 1 to mira0437250-5 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 2 to mira0437250-5 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 3 to mira0437250-5 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 5 to mira0437250-5 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: finishing write tid 6 to mira0437250-5 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 5 v 1163 (ObjNum 1738 snap 459 seq_num 1738) dirty exists 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3905: done (1 left) 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3906: done (0 left) 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: read oid 318 snap 447 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: write oid 334 current snap is 459 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: seq_num 1740 ranges {659362=565061,1712223=622744,3059361=1} 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0437250-334 from 659362 to 1224423 tid 1 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0437250-334 from 1712223 to 2334967 tid 2 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: writing mira0437250-334 from 3059361 to 3059362 tid 3 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: delete oid 227 current snap is 459 2014-07-01 17:16:06,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: expect (ObjNum 1048 snap 207 seq_num 1048) 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 1 to mira0437250-334 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 2 to mira0437250-334 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 3 to mira0437250-334 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 5 to mira0437250-334 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: finishing write tid 6 to mira0437250-334 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 334 v 1310 (ObjNum 1740 snap 459 seq_num 1740) dirty exists 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3907: done (2 left) 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3908: done (1 left) 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3909: done (0 left) 2014-07-01 17:16:06,334.334 INFO:teuthology.task.rados.rados.0.mira043.stdout:3910: snap_create 2014-07-01 17:16:07,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3910: done (0 left) 2014-07-01 17:16:07,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: read oid 172 snap -1 2014-07-01 17:16:07,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3912: read oid 235 snap 407 2014-07-01 17:16:07,081.081 INFO:teuthology.task.rados.rados.0.mira043.stdout:3913: snap_remove snap 417 2014-07-01 17:16:07,119.119 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: expect (ObjNum 1381 snap 313 seq_num 1381) 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: finishing write tid 1 to mira0435601-51 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: finishing write tid 2 to mira0435601-51 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: finishing write tid 3 to mira0435601-51 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: finishing write tid 5 to mira0435601-51 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: finishing write tid 6 to mira0435601-51 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 51 v 270 (ObjNum 2428 snap 0 seq_num 2428) dirty exists 2014-07-01 17:16:08,002.002 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: expect (ObjNum 1472 snap 0 seq_num 1472) 2014-07-01 17:16:08,086.086 INFO:teuthology.task.rados.rados.0.mira043.stdout:3911: done (2 left) 2014-07-01 17:16:08,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:3912: done (1 left) 2014-07-01 17:16:08,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:3913: done (0 left) 2014-07-01 17:16:08,087.087 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: snap_remove snap 373 2014-07-01 17:16:08,243.243 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: finishing write tid 1 to mira0435601-273 2014-07-01 17:16:08,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: finishing write tid 3 to mira0435601-273 2014-07-01 17:16:08,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: finishing write tid 4 to mira0435601-273 2014-07-01 17:16:08,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 205 (ObjNum 2429 snap 0 seq_num 2429) dirty exists 2014-07-01 17:16:08,244.244 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: expect (ObjNum 2140 snap 0 seq_num 2140) 2014-07-01 17:16:08,825.825 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:16:09,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3914: done (0 left) 2014-07-01 17:16:09,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3915: snap_create 2014-07-01 17:16:09,498.498 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 16, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.006822', u'skew': u'0.001481', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003756', u'skew': u'0.001463', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:15:45.737012', u'name': u'c', u'avail_percent': 92, u'kb_total': 944797076, u'kb_avail': 877940936, u'health': u'HEALTH_OK', u'kb_used': 18840116, u'store_stats': {u'bytes_total': 6022143, u'bytes_log': 458752, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5497839}}, {u'last_updated': u'2014-07-01 08:15:56.014965', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882513032, u'health': u'HEALTH_OK', u'kb_used': 14268020, u'store_stats': {u'bytes_total': 8642895, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5497151}}, {u'last_updated': u'2014-07-01 08:15:13.318599', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882634628, u'health': u'HEALTH_OK', u'kb_used': 14146424, u'store_stats': {u'bytes_total': 17727459, u'bytes_log': 12517376, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5144531}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'5 requests are blocked > 32 sec'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool data pg_num 44 > pgp_num 34'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool unique_pool_0 pg_num 86 > pgp_num 76'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'num_pgs': 554, u'data_bytes': 2680567498, u'bytes_used': 6326075392, u'version': 3187, u'pgs_by_state': [{u'count': 554, u'state_name': u'active+clean'}], u'bytes_avail': 5991972970496}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1614, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:16:09,498.498 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:16:09,498.498 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:09,498.498 INFO:teuthology.task.thrashosds.thrasher:Removing osd 5, in_osds are: [4, 0, 2, 3, 5, 1] 2014-07-01 17:16:09,499.499 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd out 5' 2014-07-01 17:16:09,869.869 INFO:teuthology.task.rados.rados.0.mira043.stdout:3978: done (7 left) 2014-07-01 17:16:09,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: done (6 left) 2014-07-01 17:16:09,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3980: done (5 left) 2014-07-01 17:16:09,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: done (4 left) 2014-07-01 17:16:09,870.870 INFO:teuthology.task.rados.rados.0.mira043.stdout:3982: done (3 left) 2014-07-01 17:16:09,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: done (2 left) 2014-07-01 17:16:09,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: done (1 left) 2014-07-01 17:16:09,871.871 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: done (0 left) 2014-07-01 17:16:09,872.872 INFO:teuthology.task.rados.rados.0.mira043.stdout:3986: read oid 208 snap -1 2014-07-01 17:16:10,035.035 INFO:teuthology.task.rados.rados.0.mira043.stdout:3987: read oid 268 snap -1 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 37 current snap is 0 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: seq_num 2430 ranges {2379776=602112,2981888=401408} 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: writing mira0435601-37 from 2379776 to 2981888 tid 1 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: writing mira0435601-37 from 2981888 to 3383296 tid 2 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 74 current snap is 0 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: seq_num 2431 ranges {4796416=630784,5427200=688128,6115328=573440,6688768=344064} 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: writing mira0435601-74 from 4796416 to 5427200 tid 1 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: writing mira0435601-74 from 5427200 to 6115328 tid 2 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: writing mira0435601-74 from 6115328 to 6688768 tid 3 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: writing mira0435601-74 from 6688768 to 7032832 tid 4 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 183 current snap is 0 2014-07-01 17:16:10,036.036 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: seq_num 2432 ranges {1368064=499712,1867776=520192,2387968=696320,3084288=389120} 2014-07-01 17:16:10,037.037 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: writing mira0435601-183 from 1368064 to 1867776 tid 1 2014-07-01 17:16:10,050.050 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: writing mira0435601-183 from 1867776 to 2387968 tid 2 2014-07-01 17:16:10,068.068 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: writing mira0435601-183 from 2387968 to 3084288 tid 3 2014-07-01 17:16:10,080.080 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: writing mira0435601-183 from 3084288 to 3473408 tid 4 2014-07-01 17:16:10,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3991: read oid 398 snap -1 2014-07-01 17:16:10,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: finishing write tid 1 to mira0435601-37 2014-07-01 17:16:10,201.201 INFO:teuthology.task.rados.rados.0.mira043.stdout:3986: expect (ObjNum 2022 snap 0 seq_num 2022) 2014-07-01 17:16:10,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3915: done (0 left) 2014-07-01 17:16:10,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3916: delete oid 236 current snap is 461 2014-07-01 17:16:10,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3916: done (0 left) 2014-07-01 17:16:10,202.202 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: read oid 202 snap -1 2014-07-01 17:16:10,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:3918: snap_create 2014-07-01 17:16:10,203.203 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: expect (ObjNum 1449 snap 349 seq_num 1449) 2014-07-01 17:16:11,114.114 INFO:teuthology.task.rados.rados.0.mira043.stdout:3917: done (1 left) 2014-07-01 17:16:11,115.115 INFO:teuthology.task.rados.rados.0.mira043.stdout:3918: done (0 left) 2014-07-01 17:16:11,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: read oid 368 snap 357 2014-07-01 17:16:11,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: write oid 56 current snap is 462 2014-07-01 17:16:11,117.117 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: seq_num 1743 ranges {760602=1} 2014-07-01 17:16:11,136.136 INFO:teuthology.orchestra.run.mira079.stderr:marked out osd.5. 2014-07-01 17:16:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: writing mira0437250-56 from 760602 to 760603 tid 1 2014-07-01 17:16:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: read oid 97 snap -1 2014-07-01 17:16:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: read oid 149 snap 410 2014-07-01 17:16:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: write oid 463 current snap is 462 2014-07-01 17:16:11,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: seq_num 1744 ranges {705433=639798,1505432=1} 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: writing mira0437250-463 from 705433 to 1345231 tid 1 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: writing mira0437250-463 from 1505432 to 1505433 tid 2 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: read oid 435 snap -1 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: write oid 274 current snap is 462 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: seq_num 1745 ranges {515991=611602,1715990=1} 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: writing mira0437250-274 from 515991 to 1127593 tid 1 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: writing mira0437250-274 from 1715990 to 1715991 tid 2 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: read oid 44 snap -1 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3927: delete oid 300 current snap is 462 2014-07-01 17:16:11,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: expect (ObjNum 700 snap 92 seq_num 700) 2014-07-01 17:16:11,727.727 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: expect (ObjNum 1560 snap 402 seq_num 1560) 2014-07-01 17:16:11,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: finishing write tid 1 to mira0437250-56 2014-07-01 17:16:11,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: finishing write tid 3 to mira0437250-56 2014-07-01 17:16:11,731.731 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: finishing write tid 4 to mira0437250-56 2014-07-01 17:16:11,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 56 v 1988 (ObjNum 1743 snap 462 seq_num 1743) dirty exists 2014-07-01 17:16:11,732.732 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: expect (ObjNum 1602 snap 414 seq_num 1602) 2014-07-01 17:16:11,983.983 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: finishing write tid 2 to mira0435601-37 2014-07-01 17:16:11,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: finishing write tid 4 to mira0435601-37 2014-07-01 17:16:11,984.984 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: finishing write tid 5 to mira0435601-37 2014-07-01 17:16:11,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 37 v 144 (ObjNum 2430 snap 0 seq_num 2430) dirty exists 2014-07-01 17:16:11,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 1 to mira0435601-74 2014-07-01 17:16:11,985.985 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 2 to mira0435601-74 2014-07-01 17:16:11,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 3 to mira0435601-74 2014-07-01 17:16:11,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 4 to mira0435601-74 2014-07-01 17:16:11,986.986 INFO:teuthology.task.rados.rados.0.mira043.stdout:3986: done (5 left) 2014-07-01 17:16:11,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3987: done (4 left) 2014-07-01 17:16:11,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: done (3 left) 2014-07-01 17:16:11,987.987 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: read oid 415 snap -1 2014-07-01 17:16:11,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 6 to mira0435601-74 2014-07-01 17:16:11,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: finishing write tid 7 to mira0435601-74 2014-07-01 17:16:11,988.988 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 74 v 393 (ObjNum 2431 snap 0 seq_num 2431) dirty exists 2014-07-01 17:16:11,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 1 to mira0435601-183 2014-07-01 17:16:11,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: done (3 left) 2014-07-01 17:16:11,989.989 INFO:teuthology.task.rados.rados.0.mira043.stdout:3993: read oid 84 snap -1 2014-07-01 17:16:12,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 31 current snap is 0 2014-07-01 17:16:12,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: seq_num 2433 ranges {7811072=774144,8585216=503808,9089024=520192,9609216=180224} 2014-07-01 17:16:12,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: writing mira0435601-31 from 7811072 to 8585216 tid 1 2014-07-01 17:16:12,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: writing mira0435601-31 from 8585216 to 9089024 tid 2 2014-07-01 17:16:12,150.150 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: writing mira0435601-31 from 9089024 to 9609216 tid 3 2014-07-01 17:16:12,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: writing mira0435601-31 from 9609216 to 9789440 tid 4 2014-07-01 17:16:12,160.160 INFO:teuthology.task.rados.rados.0.mira043.stdout:3995: read oid 29 snap -1 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 2 to mira0435601-183 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: read oid 214 snap -1 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 3 to mira0435601-183 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 4 to mira0435601-183 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 6 to mira0435601-183 2014-07-01 17:16:12,161.161 INFO:teuthology.task.rados.rados.0.mira043.stdout:3991: expect (ObjNum 2406 snap 0 seq_num 2406) 2014-07-01 17:16:12,651.651 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: finishing write tid 1 to mira0437250-274 2014-07-01 17:16:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: finishing write tid 2 to mira0437250-274 2014-07-01 17:16:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: finishing write tid 4 to mira0437250-274 2014-07-01 17:16:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: finishing write tid 5 to mira0437250-274 2014-07-01 17:16:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 274 v 1226 (ObjNum 1745 snap 462 seq_num 1745) dirty exists 2014-07-01 17:16:12,652.652 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: expect (ObjNum 1580 snap 407 seq_num 1580) 2014-07-01 17:16:13,128.128 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: expect (ObjNum 1334 snap 308 seq_num 1334) 2014-07-01 17:16:13,712.712 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: finishing write tid 1 to mira0437250-463 2014-07-01 17:16:13,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: finishing write tid 2 to mira0437250-463 2014-07-01 17:16:13,713.713 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: finishing write tid 4 to mira0437250-463 2014-07-01 17:16:13,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: finishing write tid 5 to mira0437250-463 2014-07-01 17:16:13,714.714 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 463 v 1651 (ObjNum 1744 snap 462 seq_num 1744) dirty exists 2014-07-01 17:16:14,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: finishing write tid 7 to mira0435601-183 2014-07-01 17:16:14,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 183 v 230 (ObjNum 2432 snap 0 seq_num 2432) dirty exists 2014-07-01 17:16:14,311.311 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: expect (ObjNum 2379 snap 0 seq_num 2379) 2014-07-01 17:16:14,637.637 INFO:teuthology.task.rados.rados.0.mira043.stdout:3919: done (8 left) 2014-07-01 17:16:14,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3920: done (7 left) 2014-07-01 17:16:14,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3921: done (6 left) 2014-07-01 17:16:14,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3922: done (5 left) 2014-07-01 17:16:14,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3923: done (4 left) 2014-07-01 17:16:14,638.638 INFO:teuthology.task.rados.rados.0.mira043.stdout:3924: done (3 left) 2014-07-01 17:16:14,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3925: done (2 left) 2014-07-01 17:16:14,639.639 INFO:teuthology.task.rados.rados.0.mira043.stdout:3926: done (1 left) 2014-07-01 17:16:14,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3927: done (0 left) 2014-07-01 17:16:14,640.640 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: delete oid 222 current snap is 462 2014-07-01 17:16:14,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3928: done (0 left) 2014-07-01 17:16:14,802.802 INFO:teuthology.task.rados.rados.0.mira043.stdout:3929: delete oid 434 current snap is 462 2014-07-01 17:16:16,148.148 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:16,149.149 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:16,150.150 INFO:teuthology.task.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.597891 2014-07-01 17:16:16,150.150 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd primary-affinity 1 0.597890786254' 2014-07-01 17:16:16,500.500 INFO:teuthology.task.rados.rados.0.mira043.stdout:3993: expect (ObjNum 2280 snap 0 seq_num 2280) 2014-07-01 17:16:17,172.172 INFO:teuthology.orchestra.run.mira079.stderr:set osd.1 primary-affinity to 0.597891 (8391832) 2014-07-01 17:16:18,703.703 INFO:teuthology.task.rados.rados.0.mira043.stdout:3929: done (0 left) 2014-07-01 17:16:18,704.704 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: snap_create 2014-07-01 17:16:19,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3995: expect (ObjNum 1939 snap 0 seq_num 1939) 2014-07-01 17:16:20,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3930: done (0 left) 2014-07-01 17:16:20,058.058 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: write oid 467 current snap is 463 2014-07-01 17:16:20,059.059 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: seq_num 1749 ranges {597134=535940,1894229=543845,2597133=1} 2014-07-01 17:16:20,076.076 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: writing mira0437250-467 from 597134 to 1133074 tid 1 2014-07-01 17:16:20,097.097 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: writing mira0437250-467 from 1894229 to 2438074 tid 2 2014-07-01 17:16:20,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: writing mira0437250-467 from 2597133 to 2597134 tid 3 2014-07-01 17:16:20,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: snap_remove snap 449 2014-07-01 17:16:20,494.494 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: finishing write tid 1 to mira0437250-467 2014-07-01 17:16:20,505.505 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: finishing write tid 2 to mira0437250-467 2014-07-01 17:16:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: finishing write tid 3 to mira0437250-467 2014-07-01 17:16:20,506.506 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: finishing write tid 5 to mira0437250-467 2014-07-01 17:16:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: finishing write tid 6 to mira0437250-467 2014-07-01 17:16:20,507.507 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 467 v 1656 (ObjNum 1749 snap 463 seq_num 1749) dirty exists 2014-07-01 17:16:21,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3931: done (1 left) 2014-07-01 17:16:21,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3932: done (0 left) 2014-07-01 17:16:21,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: read oid 211 snap -1 2014-07-01 17:16:21,098.098 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: read oid 15 snap 460 2014-07-01 17:16:21,099.099 INFO:teuthology.task.rados.rados.0.mira043.stdout:3935: snap_remove snap 445 2014-07-01 17:16:21,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: expect (ObjNum 1704 snap 446 seq_num 1704) 2014-07-01 17:16:21,449.449 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: expect (ObjNum 1441 snap 0 seq_num 1441) 2014-07-01 17:16:22,105.105 INFO:teuthology.task.rados.rados.0.mira043.stdout:3934: done (2 left) 2014-07-01 17:16:22,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3935: done (1 left) 2014-07-01 17:16:22,106.106 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: rollback oid 428 current snap is 463 2014-07-01 17:16:22,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 428 to 458 2014-07-01 17:16:22,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:3937: rollback oid 284 current snap is 463 2014-07-01 17:16:22,107.107 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 284 to 423 2014-07-01 17:16:22,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: write oid 346 current snap is 463 2014-07-01 17:16:22,108.108 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: seq_num 1750 ranges {505868=400000} 2014-07-01 17:16:22,184.184 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:22,185.185 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:22,185.185 INFO:teuthology.task.thrashosds.thrasher:Growing pool metadata 2014-07-01 17:16:22,185.185 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:16:22,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: writing mira0437250-346 from 505868 to 905868 tid 1 2014-07-01 17:16:22,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3939: delete oid 374 current snap is 463 2014-07-01 17:16:22,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 428 v 970 (ObjNum 1158 snap 238 seq_num 1158) dirty exists 2014-07-01 17:16:22,760.760 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:16:22,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3936: done (4 left) 2014-07-01 17:16:22,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3939: done (3 left) 2014-07-01 17:16:22,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: write oid 94 current snap is 463 2014-07-01 17:16:22,810.810 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: seq_num 1752 ranges {297350=1} 2014-07-01 17:16:22,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: writing mira0437250-94 from 297350 to 297351 tid 1 2014-07-01 17:16:22,811.811 INFO:teuthology.task.rados.rados.0.mira043.stdout:3941: snap_remove snap 457 2014-07-01 17:16:22,831.831 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 284 v 1903 (ObjNum 781 snap 124 seq_num 781) dirty exists 2014-07-01 17:16:22,972.972 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 1 to mira0437250-346 2014-07-01 17:16:22,975.975 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 3 to mira0437250-346 2014-07-01 17:16:22,976.976 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 1 to mira0437250-94 2014-07-01 17:16:22,977.977 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 3 to mira0437250-94 2014-07-01 17:16:22,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: finishing write tid 4 to mira0437250-94 2014-07-01 17:16:22,996.996 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 94 v 1960 (ObjNum 1752 snap 463 seq_num 1752) dirty exists 2014-07-01 17:16:23,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: finishing write tid 4 to mira0437250-346 2014-07-01 17:16:23,148.148 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 346 v 1391 (ObjNum 1750 snap 463 seq_num 1750) dirty exists 2014-07-01 17:16:23,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:3933: done (4 left) 2014-07-01 17:16:23,197.197 INFO:teuthology.task.rados.rados.0.mira043.stdout:3937: done (3 left) 2014-07-01 17:16:23,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3938: done (2 left) 2014-07-01 17:16:23,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3940: done (1 left) 2014-07-01 17:16:23,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3941: done (0 left) 2014-07-01 17:16:23,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: read oid 215 snap 460 2014-07-01 17:16:23,198.198 INFO:teuthology.task.rados.rados.0.mira043.stdout:3943: delete oid 130 current snap is 463 2014-07-01 17:16:23,337.337 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: expect (ObjNum 979 snap 181 seq_num 979) 2014-07-01 17:16:25,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3942: done (1 left) 2014-07-01 17:16:25,177.177 INFO:teuthology.task.rados.rados.0.mira043.stdout:3943: done (0 left) 2014-07-01 17:16:25,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: write oid 464 current snap is 463 2014-07-01 17:16:25,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: seq_num 1754 ranges {702339=617553,1840249=462090} 2014-07-01 17:16:25,190.190 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: writing mira0437250-464 from 702339 to 1319892 tid 1 2014-07-01 17:16:25,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 1 to mira0435601-31 2014-07-01 17:16:25,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 2 to mira0435601-31 2014-07-01 17:16:25,199.199 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 3 to mira0435601-31 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 4 to mira0435601-31 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 6 to mira0435601-31 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: finishing write tid 7 to mira0435601-31 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 361 (ObjNum 2433 snap 0 seq_num 2433) dirty exists 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: done (6 left) 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3991: done (5 left) 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: done (4 left) 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3993: done (3 left) 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: done (2 left) 2014-07-01 17:16:25,200.200 INFO:teuthology.task.rados.rados.0.mira043.stdout:3995: done (1 left) 2014-07-01 17:16:25,344.344 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: done (0 left) 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 292 current snap is 0 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: seq_num 2434 ranges {6819840=802816} 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: writing mira0435601-292 from 6819840 to 7622656 tid 1 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 69 current snap is 0 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: seq_num 2435 ranges {1572864=720896,2293760=671744,2965504=131072} 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: writing mira0435601-69 from 1572864 to 2293760 tid 1 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: writing mira0435601-69 from 2293760 to 2965504 tid 2 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: writing mira0435601-69 from 2965504 to 3096576 tid 3 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:append oid 97 current snap is 0 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: seq_num 2436 ranges {1261568=593920,1855488=499712,2355200=303104} 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: writing mira0435601-97 from 1261568 to 1855488 tid 1 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: writing mira0437250-464 from 1840249 to 2302339 tid 2 2014-07-01 17:16:25,345.345 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: delete oid 422 current snap is 463 2014-07-01 17:16:25,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: writing mira0435601-97 from 1855488 to 2355200 tid 2 2014-07-01 17:16:25,364.364 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: writing mira0435601-97 from 2355200 to 2658304 tid 3 2014-07-01 17:16:25,403.403 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 3 2014-07-01 17:16:25,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3945: done (1 left) 2014-07-01 17:16:25,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: read oid 116 snap 450 2014-07-01 17:16:25,511.511 INFO:teuthology.task.rados.rados.0.mira043.stdout:3947: snap_remove snap 446 2014-07-01 17:16:25,519.519 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: expect (ObjNum 1228 snap 264 seq_num 1228) 2014-07-01 17:16:26,529.529 INFO:teuthology.task.thrashosds.ceph_manager:increase pool size by 10 2014-07-01 17:16:26,529.529 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool set metadata pg_num 74' 2014-07-01 17:16:26,599.599 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 1 to mira0435601-292 2014-07-01 17:16:26,670.670 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 3 to mira0435601-292 2014-07-01 17:16:26,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 4 to mira0435601-292 2014-07-01 17:16:26,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 292 v 312 (ObjNum 2434 snap 0 seq_num 2434) dirty exists 2014-07-01 17:16:26,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: done (2 left) 2014-07-01 17:16:26,673.673 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 2 2014-07-01 17:16:26,695.695 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: finishing write tid 1 to mira0437250-464 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: finishing write tid 2 to mira0437250-464 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: finishing write tid 4 to mira0437250-464 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: finishing write tid 5 to mira0437250-464 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 464 v 1365 (ObjNum 1754 snap 463 seq_num 1754) dirty exists 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3944: done (2 left) 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3946: done (1 left) 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3947: done (0 left) 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3948: snap_create 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: finishing write tid 1 to mira0435601-97 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: finishing write tid 1 to mira0435601-69 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: finishing write tid 2 to mira0435601-97 2014-07-01 17:16:26,764.764 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: finishing write tid 3 to mira0435601-97 2014-07-01 17:16:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: finishing write tid 5 to mira0435601-97 2014-07-01 17:16:26,835.835 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: finishing write tid 6 to mira0435601-97 2014-07-01 17:16:26,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 97 v 364 (ObjNum 2436 snap 0 seq_num 2436) dirty exists 2014-07-01 17:16:26,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: done (1 left) 2014-07-01 17:16:26,836.836 INFO:teuthology.task.rados.rados.0.mira043.stdout: waiting on 1 2014-07-01 17:16:27,253.253 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: finishing write tid 2 to mira0435601-69 2014-07-01 17:16:27,255.255 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: finishing write tid 3 to mira0435601-69 2014-07-01 17:16:27,256.256 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: finishing write tid 5 to mira0435601-69 2014-07-01 17:16:27,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: finishing write tid 6 to mira0435601-69 2014-07-01 17:16:27,271.271 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 69 v 289 (ObjNum 2435 snap 0 seq_num 2435) dirty exists 2014-07-01 17:16:27,272.272 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: done (0 left) 2014-07-01 17:16:27,419.419 INFO:teuthology.task.rados.rados.0.mira043.stderr:0 errors. 2014-07-01 17:16:27,419.419 INFO:teuthology.task.rados.rados.0.mira043.stderr: 2014-07-01 17:16:27,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3948: done (0 left) 2014-07-01 17:16:27,728.728 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: snap_create 2014-07-01 17:16:27,738.738 INFO:teuthology.orchestra.run.mira079.stderr:set pool 1 pg_num to 74 2014-07-01 17:16:27,756.756 INFO:teuthology.task.thrashosds.ceph_manager:removing pool_name unique_pool_0 2014-07-01 17:16:27,756.756 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados rmpool unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it' 2014-07-01 17:16:28,730.730 INFO:teuthology.orchestra.run.mira079.stdout:successfully deleted pool unique_pool_0 2014-07-01 17:16:28,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3949: done (0 left) 2014-07-01 17:16:28,730.730 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: snap_remove snap 461 2014-07-01 17:16:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3950: done (0 left) 2014-07-01 17:16:29,736.736 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: delete oid 264 current snap is 465 2014-07-01 17:16:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3951: done (0 left) 2014-07-01 17:16:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: write oid 31 current snap is 465 2014-07-01 17:16:29,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: seq_num 1757 ranges {686204=533990,1843701=642302,3041776=444428} 2014-07-01 17:16:29,824.824 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: writing mira0437250-31 from 686204 to 1220194 tid 1 2014-07-01 17:16:29,847.847 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: writing mira0437250-31 from 1843701 to 2486003 tid 2 2014-07-01 17:16:29,863.863 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: writing mira0437250-31 from 3041776 to 3486204 tid 3 2014-07-01 17:16:29,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3953: delete oid 197 current snap is 465 2014-07-01 17:16:29,900.900 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 1 to mira0437250-31 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 2 to mira0437250-31 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3953: done (1 left) 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: read oid 134 snap 435 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3955: snap_remove snap 433 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 3 to mira0437250-31 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 5 to mira0437250-31 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: finishing write tid 6 to mira0437250-31 2014-07-01 17:16:29,901.901 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 31 v 1993 (ObjNum 1757 snap 465 seq_num 1757) dirty exists 2014-07-01 17:16:29,954.954 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: expect (ObjNum 1527 snap 381 seq_num 1527) 2014-07-01 17:16:30,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3952: done (2 left) 2014-07-01 17:16:30,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3954: done (1 left) 2014-07-01 17:16:30,790.790 INFO:teuthology.task.rados.rados.0.mira043.stdout:3955: done (0 left) 2014-07-01 17:16:30,791.791 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: snap_remove snap 447 2014-07-01 17:16:31,765.765 INFO:teuthology.task.rados.rados.0.mira043.stdout:3956: done (0 left) 2014-07-01 17:16:31,766.766 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: snap_create 2014-07-01 17:16:32,756.756 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:32,756.756 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:32,756.756 INFO:teuthology.task.thrashosds.thrasher:fixing pg num pool metadata 2014-07-01 17:16:32,756.756 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:16:32,768.768 INFO:teuthology.task.rados.rados.0.mira043.stdout:3957: done (0 left) 2014-07-01 17:16:32,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: write oid 433 current snap is 466 2014-07-01 17:16:32,769.769 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: seq_num 1759 ranges {692215=512671,1492214=1} 2014-07-01 17:16:32,794.794 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: writing mira0437250-433 from 692215 to 1204886 tid 1 2014-07-01 17:16:32,797.797 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: writing mira0437250-433 from 1492214 to 1492215 tid 2 2014-07-01 17:16:32,798.798 INFO:teuthology.task.rados.rados.0.mira043.stdout:3959: delete oid 424 current snap is 466 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3959: done (1 left) 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: rollback oid 182 current snap is 466 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 182 to 455 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3961: rollback oid 310 current snap is 466 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 310 to 415 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: snap_remove snap 430 2014-07-01 17:16:32,934.934 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 182 v 1664 (ObjNum 1544 snap 392 seq_num 1544) dirty exists 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 310 v 923 (ObjNum 1247 snap 272 seq_num 1247) dirty exists 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 1 to mira0437250-433 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 2 to mira0437250-433 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 4 to mira0437250-433 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: finishing write tid 5 to mira0437250-433 2014-07-01 17:16:32,935.935 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 433 v 897 (ObjNum 1759 snap 466 seq_num 1759) dirty exists 2014-07-01 17:16:33,491.491 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:16:33,778.778 INFO:teuthology.task.rados.rados.0.mira043.stdout:3958: done (3 left) 2014-07-01 17:16:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3960: done (2 left) 2014-07-01 17:16:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3961: done (1 left) 2014-07-01 17:16:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3962: done (0 left) 2014-07-01 17:16:33,779.779 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: delete oid 484 current snap is 466 2014-07-01 17:16:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3963: done (0 left) 2014-07-01 17:16:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: write oid 75 current snap is 466 2014-07-01 17:16:33,864.864 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: seq_num 1762 ranges {426480=615604,1576218=50262} 2014-07-01 17:16:33,888.888 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: writing mira0437250-75 from 426480 to 1042084 tid 1 2014-07-01 17:16:33,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: writing mira0437250-75 from 1576218 to 1626480 tid 2 2014-07-01 17:16:33,894.894 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: delete oid 328 current snap is 466 2014-07-01 17:16:33,942.942 INFO:teuthology.task.rados.rados.0.mira043.stdout:3965: done (1 left) 2014-07-01 17:16:33,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: write oid 268 current snap is 466 2014-07-01 17:16:33,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: seq_num 1764 ranges {583916=711661,1886787=697129} 2014-07-01 17:16:33,943.943 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: writing mira0437250-268 from 583916 to 1295577 tid 1 2014-07-01 17:16:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: writing mira0437250-268 from 1886787 to 2583916 tid 2 2014-07-01 17:16:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: read oid 367 snap -1 2014-07-01 17:16:34,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: finishing write tid 1 to mira0437250-75 2014-07-01 17:16:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: finishing write tid 2 to mira0437250-75 2014-07-01 17:16:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: finishing write tid 4 to mira0437250-75 2014-07-01 17:16:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: finishing write tid 5 to mira0437250-75 2014-07-01 17:16:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 75 v 1111 (ObjNum 1762 snap 466 seq_num 1762) dirty exists 2014-07-01 17:16:34,055.055 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 1 to mira0437250-268 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3964: done (2 left) 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3968: rollback oid 298 current snap is 466 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 298 to 435 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: read oid 154 snap -1 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3970: snap_create 2014-07-01 17:16:34,056.056 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: expect (ObjNum 153 snap 0 seq_num 153) 2014-07-01 17:16:35,298.298 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: expect (ObjNum 1732 snap 452 seq_num 1732) 2014-07-01 17:16:36,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 2 to mira0437250-268 2014-07-01 17:16:36,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 4 to mira0437250-268 2014-07-01 17:16:36,991.991 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: finishing write tid 5 to mira0437250-268 2014-07-01 17:16:36,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 268 v 902 (ObjNum 1764 snap 466 seq_num 1764) dirty exists 2014-07-01 17:16:36,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 298 v 1382 (ObjNum 4000000 snap 0 seq_num 400000) dirty dne 2014-07-01 17:16:36,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3966: done (4 left) 2014-07-01 17:16:36,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3967: done (3 left) 2014-07-01 17:16:36,992.992 INFO:teuthology.task.rados.rados.0.mira043.stdout:3968: done (2 left) 2014-07-01 17:16:36,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3969: done (1 left) 2014-07-01 17:16:36,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3970: done (0 left) 2014-07-01 17:16:36,993.993 INFO:teuthology.task.rados.rados.0.mira043.stdout:3971: read oid 121 snap -1 2014-07-01 17:16:36,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:3972: rollback oid 492 current snap is 467 2014-07-01 17:16:36,994.994 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 492 to 454 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: write oid 20 current snap is 467 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: seq_num 1765 ranges {527722=683465,1848471=590360,3148733=178989} 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: writing mira0437250-20 from 527722 to 1211187 tid 1 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: writing mira0437250-20 from 1848471 to 2438831 tid 2 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: writing mira0437250-20 from 3148733 to 3327722 tid 3 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: snap_create 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 492 v 1608 (ObjNum 1531 snap 384 seq_num 1531) dirty exists 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: finishing write tid 1 to mira0437250-20 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: finishing write tid 2 to mira0437250-20 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: finishing write tid 3 to mira0437250-20 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: finishing write tid 5 to mira0437250-20 2014-07-01 17:16:37,156.156 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: finishing write tid 6 to mira0437250-20 2014-07-01 17:16:37,157.157 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 20 v 1396 (ObjNum 1765 snap 467 seq_num 1765) dirty exists 2014-07-01 17:16:38,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3971: done (3 left) 2014-07-01 17:16:38,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3972: done (2 left) 2014-07-01 17:16:38,008.008 INFO:teuthology.task.rados.rados.0.mira043.stdout:3973: done (1 left) 2014-07-01 17:16:38,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3974: done (0 left) 2014-07-01 17:16:38,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: write oid 376 current snap is 468 2014-07-01 17:16:38,009.009 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: seq_num 1766 ranges {72550=1} 2014-07-01 17:16:38,010.010 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: writing mira0437250-376 from 72550 to 72551 tid 1 2014-07-01 17:16:38,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: read oid 324 snap -1 2014-07-01 17:16:38,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: write oid 206 current snap is 468 2014-07-01 17:16:38,011.011 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: seq_num 1767 ranges {567781=511746,1772863=633598,3034831=628164,3767780=1} 2014-07-01 17:16:38,029.029 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: writing mira0437250-206 from 567781 to 1079527 tid 1 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: writing mira0437250-206 from 1772863 to 2406461 tid 2 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: writing mira0437250-206 from 3034831 to 3662995 tid 3 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: writing mira0437250-206 from 3767780 to 3767781 tid 4 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3978: read oid 400 snap -1 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 1 to mira0437250-376 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 3 to mira0437250-376 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: finishing write tid 4 to mira0437250-376 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 376 v 1992 (ObjNum 1766 snap 468 seq_num 1766) dirty exists 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 1 to mira0437250-206 2014-07-01 17:16:38,178.178 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 2 to mira0437250-206 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3975: done (3 left) 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3976: done (2 left) 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: write oid 454 current snap is 468 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: seq_num 1768 ranges {628963=400000} 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: writing mira0437250-454 from 628963 to 1028963 tid 1 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3980: snap_create 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 3 to mira0437250-206 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 4 to mira0437250-206 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 6 to mira0437250-206 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: finishing write tid 7 to mira0437250-206 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 206 v 1685 (ObjNum 1767 snap 468 seq_num 1767) dirty exists 2014-07-01 17:16:38,179.179 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: finishing write tid 1 to mira0437250-454 2014-07-01 17:16:38,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: finishing write tid 3 to mira0437250-454 2014-07-01 17:16:38,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: finishing write tid 4 to mira0437250-454 2014-07-01 17:16:38,180.180 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 454 v 1385 (ObjNum 1768 snap 468 seq_num 1768) dirty exists 2014-07-01 17:16:39,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:3977: done (3 left) 2014-07-01 17:16:39,052.052 INFO:teuthology.task.rados.rados.0.mira043.stdout:3978: done (2 left) 2014-07-01 17:16:39,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3979: done (1 left) 2014-07-01 17:16:39,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3980: done (0 left) 2014-07-01 17:16:39,053.053 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: write oid 122 current snap is 469 2014-07-01 17:16:39,054.054 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: seq_num 1769 ranges {573793=607803,1773792=1} 2014-07-01 17:16:39,075.075 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: writing mira0437250-122 from 573793 to 1181596 tid 1 2014-07-01 17:16:39,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: writing mira0437250-122 from 1773792 to 1773793 tid 2 2014-07-01 17:16:39,079.079 INFO:teuthology.task.rados.rados.0.mira043.stdout:3982: delete oid 208 current snap is 469 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:3982: done (1 left) 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: rollback oid 189 current snap is 469 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 189 to 466 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: delete oid 424 current snap is 469 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:3984: done (2 left) 2014-07-01 17:16:39,220.220 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: read oid 244 snap -1 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3986: rollback oid 490 current snap is 469 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 490 to 435 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3987: read oid 406 snap -1 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: delete oid 404 current snap is 469 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3988: done (5 left) 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: rollback oid 273 current snap is 469 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 273 to 465 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: read oid 361 snap -1 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3991: rollback oid 61 current snap is 469 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:rollback oid 61 to 454 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: read oid 24 snap -1 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3993: snap_create 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 189 v 0 (ObjNum 17115328 snap 0 seq_num 32) dirty dne 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 61 v 1387 (ObjNum 1633 snap 421 seq_num 1633) dirty exists 2014-07-01 17:16:39,221.221 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: expect (ObjNum 1468 snap 357 seq_num 1468) 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 273 v 0 (ObjNum -1 snap 0 seq_num 0) dirty dne 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: finishing write tid 1 to mira0437250-122 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: finishing write tid 2 to mira0437250-122 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: finishing write tid 4 to mira0437250-122 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: finishing write tid 5 to mira0437250-122 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 122 v 937 (ObjNum 1769 snap 469 seq_num 1769) dirty exists 2014-07-01 17:16:39,481.481 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: expect (ObjNum 1003 snap 194 seq_num 1003) 2014-07-01 17:16:40,610.610 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: expect (ObjNum 1404 snap 323 seq_num 1404) 2014-07-01 17:16:41,682.682 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:41,683.683 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:41,683.683 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 5 2014-07-01 17:16:41,683.683 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:16:41,683.683 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:16:41,684.684 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok config set filestore_inject_stall 3' 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 490 v 680 (ObjNum 1255 snap 274 seq_num 1255) dirty exists 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3981: done (9 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3983: done (8 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3985: done (7 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3986: done (6 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3987: done (5 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3989: done (4 left) 2014-07-01 17:16:42,230.230 INFO:teuthology.task.rados.rados.0.mira043.stdout:3990: done (3 left) 2014-07-01 17:16:42,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3991: done (2 left) 2014-07-01 17:16:42,231.231 INFO:teuthology.task.rados.rados.0.mira043.stdout:3992: done (1 left) 2014-07-01 17:16:42,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:3993: done (0 left) 2014-07-01 17:16:42,232.232 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: snap_create 2014-07-01 17:16:43,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:3994: done (0 left) 2014-07-01 17:16:43,078.078 INFO:teuthology.task.rados.rados.0.mira043.stdout:3995: snap_create 2014-07-01 17:16:44,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3995: done (0 left) 2014-07-01 17:16:44,089.089 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: write oid 106 current snap is 472 2014-07-01 17:16:44,090.090 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: seq_num 1773 ranges {769240=682540,1951416=405169,2952839=216401} 2014-07-01 17:16:44,116.116 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: writing mira0437250-106 from 769240 to 1451780 tid 1 2014-07-01 17:16:44,132.132 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: writing mira0437250-106 from 1951416 to 2356585 tid 2 2014-07-01 17:16:44,142.142 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: writing mira0437250-106 from 2952839 to 3169240 tid 3 2014-07-01 17:16:44,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: write oid 224 current snap is 472 2014-07-01 17:16:44,144.144 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: seq_num 1774 ranges {714069=690440,2066571=510436,3191183=557106,3914068=1} 2014-07-01 17:16:44,283.283 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: writing mira0437250-224 from 714069 to 1404509 tid 1 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: writing mira0437250-224 from 2066571 to 2577007 tid 2 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: writing mira0437250-224 from 3191183 to 3748289 tid 3 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: writing mira0437250-224 from 3914068 to 3914069 tid 4 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: delete oid 275 current snap is 472 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: finishing write tid 1 to mira0437250-106 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: finishing write tid 2 to mira0437250-106 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: finishing write tid 3 to mira0437250-106 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: finishing write tid 5 to mira0437250-106 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: finishing write tid 6 to mira0437250-106 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 106 v 1681 (ObjNum 1773 snap 472 seq_num 1773) dirty exists 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 1 to mira0437250-224 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 2 to mira0437250-224 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3996: done (2 left) 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3998: done (1 left) 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: snap_create 2014-07-01 17:16:44,284.284 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 3 to mira0437250-224 2014-07-01 17:16:44,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 4 to mira0437250-224 2014-07-01 17:16:44,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 6 to mira0437250-224 2014-07-01 17:16:44,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: finishing write tid 7 to mira0437250-224 2014-07-01 17:16:44,285.285 INFO:teuthology.task.rados.rados.0.mira043.stdout:update_object_version oid 224 v 1772 (ObjNum 1774 snap 472 seq_num 1774) dirty exists 2014-07-01 17:16:45,095.095 INFO:teuthology.task.rados.rados.0.mira043.stdout:3997: done (1 left) 2014-07-01 17:16:45,096.096 INFO:teuthology.task.rados.rados.0.mira043.stdout:3999: done (0 left) 2014-07-01 17:16:45,098.098 INFO:teuthology.task.rados.rados.0.mira043.stderr:0 errors. 2014-07-01 17:16:45,098.098 INFO:teuthology.task.rados.rados.0.mira043.stderr: 2014-07-01 17:16:45,107.107 INFO:teuthology.task.thrashosds.ceph_manager:removing pool_name unique_pool_2 2014-07-01 17:16:45,107.107 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados rmpool unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it' 2014-07-01 17:16:46,101.101 INFO:teuthology.orchestra.run.mira079.stdout:successfully deleted pool unique_pool_2 2014-07-01 17:16:46,103.103 DEBUG:teuthology.run_tasks:Unwinding manager swift 2014-07-01 17:16:46,103.103 DEBUG:teuthology.run_tasks:Unwinding manager rgw 2014-07-01 17:16:46,104.104 INFO:teuthology.task.rgw:Stopping apache... 2014-07-01 17:16:46,277.277 INFO:teuthology.misc:Shutting down rgw daemons... 2014-07-01 17:16:46,277.277 DEBUG:teuthology.task.rgw.client.0:waiting for process to exit 2014-07-01 17:16:46,645.645 INFO:teuthology.task.rgw.client.0.mira043.stdout:2014-07-01 08:16:46.556558 7f3ec9775780 -1 shutting down 2014-07-01 17:16:47,153.153 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:47,153.153 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:47,153.153 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 3 2014-07-01 17:16:47,153.153 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:16:47,154.154 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:16:47,154.154 INFO:teuthology.orchestra.run.mira028:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok config set filestore_inject_stall 3' 2014-07-01 17:16:52,277.277 INFO:teuthology.task.rgw.client.0:Stopped 2014-07-01 17:16:52,278.278 INFO:teuthology.orchestra.run.mira043:Running: 'rm -f /home/ubuntu/cephtest/rgw.opslog.client.0.sock' 2014-07-01 17:16:52,565.565 INFO:teuthology.task.thrashosds.thrasher:in_osds: [4, 0, 2, 3, 1] out_osds: [5] dead_osds: [] live_osds: [0, 2, 4, 3, 5, 1] 2014-07-01 17:16:52,565.565 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 3 min_out 0 min_live 2 min_dead 0 2014-07-01 17:16:52,566.566 INFO:teuthology.task.thrashosds.thrasher:inject_pause on 2 2014-07-01 17:16:52,566.566 INFO:teuthology.task.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2014-07-01 17:16:52,566.566 INFO:teuthology.task.thrashosds.thrasher:Checking after 0, should_be_down=False 2014-07-01 17:16:52,566.566 INFO:teuthology.orchestra.run.mira079:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config set filestore_inject_stall 3' 2014-07-01 17:16:52,882.882 INFO:teuthology.task.rgw:Removing apache config... 2014-07-01 17:16:52,882.882 INFO:teuthology.orchestra.run.mira043:Running: 'rm -f /home/ubuntu/cephtest/apache/apache.client.0.conf && rm -f /home/ubuntu/cephtest/apache/htdocs.client.0/rgw.fcgi' 2014-07-01 17:16:53,510.510 INFO:teuthology.task.rgw:Cleaning up apache directories... 2014-07-01 17:16:53,510.510 INFO:teuthology.orchestra.run.mira043:Running: 'rm -rf /home/ubuntu/cephtest/apache/tmp.client.0 && rmdir /home/ubuntu/cephtest/apache/htdocs.client.0' 2014-07-01 17:16:54,166.166 INFO:teuthology.orchestra.run.mira043:Running: 'rmdir /home/ubuntu/cephtest/apache' 2014-07-01 17:16:54,852.852 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2014-07-01 17:16:54,853.853 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2014-07-01 17:16:54,853.853 DEBUG:teuthology.run_tasks:Unwinding manager radosbench 2014-07-01 17:16:54,853.853 INFO:teuthology.task.radosbench:joining radosbench (timing out after 9000s) 2014-07-01 17:16:54,853.853 INFO:teuthology.task.thrashosds.ceph_manager:removing pool_name unique_pool_1 2014-07-01 17:16:54,854.854 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados rmpool unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it' 2014-07-01 17:16:55,373.373 INFO:teuthology.orchestra.run.mira079.stdout:successfully deleted pool unique_pool_1 2014-07-01 17:16:55,375.375 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2014-07-01 17:16:55,375.375 DEBUG:teuthology.run_tasks:Unwinding manager rados 2014-07-01 17:16:55,375.375 INFO:teuthology.task.rados:joining rados 2014-07-01 17:16:55,375.375 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2014-07-01 17:16:55,375.375 DEBUG:teuthology.run_tasks:Unwinding manager thrashosds 2014-07-01 17:16:55,375.375 INFO:teuthology.task.thrashosds:joining thrashosds 2014-07-01 17:16:58,056.056 INFO:teuthology.task.thrashosds.thrasher:inning osd 2014-07-01 17:16:58,056.056 INFO:teuthology.task.thrashosds.thrasher:Adding osd 5 2014-07-01 17:16:58,058.058 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd in 5' 2014-07-01 17:16:59,449.449 INFO:teuthology.orchestra.run.mira079.stderr:marked in osd.5. 2014-07-01 17:16:59,462.462 INFO:teuthology.task.thrashosds.thrasher:Added osd 5 2014-07-01 17:16:59,463.463 INFO:teuthology.task.thrashosds.ceph_manager:waiting for recovery to complete 2014-07-01 17:16:59,463.463 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:17:00,473.473 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:02,896.896 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-07-01 17:17:03,719.719 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:06,016.016 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-07-01 17:17:06,813.813 INFO:teuthology.task.thrashosds.ceph_manager:{u'election_epoch': 14, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 1, u'epoch': 5, u'by_rank': [{u'status': u'up:active', u'name': u'a', u'rank': 0}], u'up': 1, u'in': 1}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'c', u'rank': 0, u'addr': u'10.214.132.120:6789/0'}, {u'name': u'a', u'rank': 1, u'addr': u'10.214.135.128:6789/0'}, {u'name': u'b', u'rank': 2, u'addr': u'10.214.135.128:6790/0'}], u'modified': u'2014-07-01 07:11:56.830090', u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d', u'created': u'2014-07-01 07:11:56.830090'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 14, u'round': 16, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}, {u'latency': u'0.006822', u'skew': u'0.001481', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003756', u'skew': u'0.001463', u'health': u'HEALTH_OK', u'name': u'b'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-07-01 08:16:45.737381', u'name': u'c', u'avail_percent': 92, u'kb_total': 944797076, u'kb_avail': 877778896, u'health': u'HEALTH_OK', u'kb_used': 19002156, u'store_stats': {u'bytes_total': 7965780, u'bytes_log': 2031616, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5868612}}, {u'last_updated': u'2014-07-01 08:16:56.015381', u'name': u'a', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882343028, u'health': u'HEALTH_OK', u'kb_used': 14438024, u'store_stats': {u'bytes_total': 9013487, u'bytes_log': 3080192, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5867743}}, {u'last_updated': u'2014-07-01 08:16:13.318910', u'name': u'b', u'avail_percent': 93, u'kb_total': 944797076, u'kb_avail': 882440436, u'health': u'HEALTH_OK', u'kb_used': 14340616, u'store_stats': {u'bytes_total': 11788630, u'bytes_log': 6225920, u'last_updated': u'0.000000', u'bytes_misc': 65552, u'bytes_sst': 5497158}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'7 pgs recovering'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 122/508 objects degraded (24.016%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool data pg_num 44 > pgp_num 34'}, {u'severity': u'HEALTH_WARN', u'summary': u'pool metadata pg_num 74 > pgp_num 64'}]}, u'pgmap': {u'bytes_total': 5998299045888, u'recovering_bytes_per_sec': 0, u'degraded_objects': 122, u'num_pgs': 326, u'recovering_keys_per_sec': 27, u'data_bytes': 1092263, u'degraded_total': 508, u'bytes_used': 1086640128, u'recovering_objects_per_sec': 0, u'version': 3236, u'pgs_by_state': [{u'count': 17, u'state_name': u'active'}, {u'count': 302, u'state_name': u'active+clean'}, {u'count': 7, u'state_name': u'active+recovering'}], u'degraded_ratio': u'24.016', u'bytes_avail': 5997212405760}, u'quorum_names': [u'c', u'a', u'b'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 6, u'num_up_osds': 6, u'epoch': 1642, u'num_in_osds': 6}}, u'fsid': u'd0c0ad56-ac30-428a-b414-dd3ce1a5627d'} 2014-07-01 17:17:06,814.814 INFO:teuthology.task.thrashosds.ceph_manager:recovered! 2014-07-01 17:17:06,814.814 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2014-07-01 17:17:06,815.815 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2014-07-01 17:17:06,815.815 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2014-07-01 17:17:06,815.815 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:17:07,634.634 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:10,119.119 INFO:teuthology.task.ceph:Waiting for all osds to be active and clean. 2014-07-01 17:17:20,104.104 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:17:20,875.875 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:22,922.922 INFO:teuthology.task.ceph:Waiting for all osds to be active and clean. 2014-07-01 17:17:32,904.904 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:17:33,722.722 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:35,882.882 INFO:teuthology.task.ceph:Waiting for all osds to be active and clean. 2014-07-01 17:17:45,861.861 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:17:46,625.625 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:17:48,671.671 INFO:teuthology.task.ceph:Waiting for all osds to be active and clean. 2014-07-01 17:17:58,651.651 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:17:59,421.421 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:18:01,651.651 INFO:teuthology.task.ceph:Waiting for all osds to be active and clean. 2014-07-01 17:18:11,628.628 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:18:12,398.398 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:18:15,582.582 INFO:teuthology.task.ceph:Scrubbing osd osd.0 2014-07-01 17:18:15,583.583 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.0' 2014-07-01 17:18:16,154.154 INFO:teuthology.orchestra.run.mira079.stderr:osd.0 instructed to scrub 2014-07-01 17:18:16,168.168 INFO:teuthology.task.ceph:Scrubbing osd osd.1 2014-07-01 17:18:16,169.169 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.1' 2014-07-01 17:18:16,935.935 INFO:teuthology.orchestra.run.mira079.stderr:osd.1 instructed to scrub 2014-07-01 17:18:16,947.947 INFO:teuthology.task.ceph:Scrubbing osd osd.2 2014-07-01 17:18:16,947.947 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.2' 2014-07-01 17:18:17,716.716 INFO:teuthology.orchestra.run.mira079.stderr:osd.2 instructed to scrub 2014-07-01 17:18:17,727.727 INFO:teuthology.task.ceph:Scrubbing osd osd.3 2014-07-01 17:18:17,728.728 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.3' 2014-07-01 17:18:18,489.489 INFO:teuthology.orchestra.run.mira079.stderr:osd.3 instructed to scrub 2014-07-01 17:18:18,502.502 INFO:teuthology.task.ceph:Scrubbing osd osd.4 2014-07-01 17:18:18,503.503 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.4' 2014-07-01 17:18:19,240.240 INFO:teuthology.orchestra.run.mira079.stderr:osd.4 instructed to scrub 2014-07-01 17:18:19,251.251 INFO:teuthology.task.ceph:Scrubbing osd osd.5 2014-07-01 17:18:19,251.251 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd scrub osd.5' 2014-07-01 17:18:19,827.827 INFO:teuthology.orchestra.run.mira079.stderr:osd.5 instructed to scrub 2014-07-01 17:18:19,842.842 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:18:20,796.796 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:18:23,011.011 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:18:32,982.982 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:18:33,716.716 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:18:35,924.924 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:18:45,904.904 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:18:46,500.500 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:18:48,884.884 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:18:58,865.865 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:18:59,461.461 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:19:01,855.855 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:19:11,825.825 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:19:12,596.596 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:19:14,657.657 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:19:24,633.633 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:19:25,401.401 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:19:27,457.457 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:19:37,431.431 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:19:37,990.990 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:19:40,231.231 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:19:50,210.210 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:19:50,965.965 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:19:53,018.018 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:20:02,994.994 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:20:03,550.550 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:20:05,937.937 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:20:15,917.917 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:20:16,512.512 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:20:18,907.907 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:20:28,881.881 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:20:29,672.672 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:20:31,734.734 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:20:41,710.710 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:20:42,480.480 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:20:44,540.540 INFO:teuthology.task.ceph:Still waiting for all pgs to be scrubbed. 2014-07-01 17:20:54,518.518 INFO:teuthology.orchestra.run.mira079:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-07-01 17:20:55,404.404 INFO:teuthology.orchestra.run.mira079.stderr:dumped all in format json 2014-07-01 17:20:58,171.171 INFO:teuthology.task.ceph:Exiting scrub checking -- not all pgs scrubbed. 2014-07-01 17:20:58,172.172 INFO:teuthology.misc:Shutting down mds daemons... 2014-07-01 17:20:58,173.173 DEBUG:teuthology.task.ceph.mds.a:waiting for process to exit 2014-07-01 17:21:04,146.146 INFO:teuthology.task.ceph.mds.a:Stopped 2014-07-01 17:21:04,146.146 INFO:teuthology.misc:Shutting down osd daemons... 2014-07-01 17:21:04,147.147 DEBUG:teuthology.task.ceph.osd.1:waiting for process to exit 2014-07-01 17:21:10,147.147 INFO:teuthology.task.ceph.osd.1:Stopped 2014-07-01 17:21:10,148.148 DEBUG:teuthology.task.ceph.osd.0:waiting for process to exit 2014-07-01 17:21:16,148.148 INFO:teuthology.task.ceph.osd.0:Stopped 2014-07-01 17:21:16,148.148 DEBUG:teuthology.task.ceph.osd.3:waiting for process to exit 2014-07-01 17:21:22,149.149 INFO:teuthology.task.ceph.osd.3:Stopped 2014-07-01 17:21:22,150.150 DEBUG:teuthology.task.ceph.osd.2:waiting for process to exit 2014-07-01 17:21:28,149.149 INFO:teuthology.task.ceph.osd.2:Stopped 2014-07-01 17:21:28,151.151 DEBUG:teuthology.task.ceph.osd.5:waiting for process to exit 2014-07-01 17:21:34,151.151 INFO:teuthology.task.ceph.osd.5:Stopped 2014-07-01 17:21:34,152.152 DEBUG:teuthology.task.ceph.osd.4:waiting for process to exit 2014-07-01 17:21:40,152.152 INFO:teuthology.task.ceph.osd.4:Stopped 2014-07-01 17:21:40,152.152 INFO:teuthology.misc:Shutting down mon daemons... 2014-07-01 17:21:40,153.153 DEBUG:teuthology.task.ceph.mon.a:waiting for process to exit 2014-07-01 17:21:46,153.153 INFO:teuthology.task.ceph.mon.a:Stopped 2014-07-01 17:21:46,154.154 DEBUG:teuthology.task.ceph.mon.c:waiting for process to exit 2014-07-01 17:21:52,155.155 INFO:teuthology.task.ceph.mon.c:Stopped 2014-07-01 17:21:52,156.156 DEBUG:teuthology.task.ceph.mon.b:waiting for process to exit 2014-07-01 17:21:58,155.155 INFO:teuthology.task.ceph.mon.b:Stopped 2014-07-01 17:21:58,156.156 INFO:teuthology.task.ceph:Checking cluster log for badness... 2014-07-01 17:21:58,156.156 INFO:teuthology.orchestra.run.mira079:Running: "sudo egrep '\\[ERR\\]|\\[WRN\\]|\\[SEC\\]' /var/log/ceph/ceph.log | egrep -v 'slow request' | egrep -v 'wrongly marked me down' | egrep -v 'objects unfound and apparently lost' | egrep -v 'log bound mismatch' | head -n 1" 2014-07-01 17:21:58,579.579 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 17:21:58,580.580 INFO:teuthology.orchestra.run.mira079:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-0' 2014-07-01 17:21:59,220.220 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 17:21:59,220.220 INFO:teuthology.orchestra.run.mira079:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-1' 2014-07-01 17:21:59,707.707 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 17:21:59,708.708 INFO:teuthology.orchestra.run.mira079:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-2' 2014-07-01 17:22:00,266.266 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 17:22:00,266.266 INFO:teuthology.orchestra.run.mira028:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-3' 2014-07-01 17:22:00,720.720 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-4 on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 17:22:00,721.721 INFO:teuthology.orchestra.run.mira028:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-4' 2014-07-01 17:22:01,251.251 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-5 on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 17:22:01,252.252 INFO:teuthology.orchestra.run.mira028:Running: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-5' 2014-07-01 17:22:01,811.811 INFO:teuthology.task.ceph:Cleaning ceph cluster... 2014-07-01 17:22:01,812.812 INFO:teuthology.orchestra.run.mira028:Running: 'sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/data /home/ubuntu/cephtest/monmap' 2014-07-01 17:22:02,284.284 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/data /home/ubuntu/cephtest/monmap' 2014-07-01 17:22:02,622.622 INFO:teuthology.orchestra.run.mira079:Running: 'sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/data /home/ubuntu/cephtest/monmap' 2014-07-01 17:22:02,996.996 INFO:teuthology.task.ceph:Checking for errors in any valgrind logs... 2014-07-01 17:22:02,997.997 INFO:teuthology.orchestra.run.mira079:Running: "sudo zgrep '' /var/log/ceph/valgrind/* /dev/null | sort | uniq" 2014-07-01 17:22:03,505.505 INFO:teuthology.orchestra.run.mira043:Running: "sudo zgrep '' /var/log/ceph/valgrind/* /dev/null | sort | uniq" 2014-07-01 17:22:03,589.589 INFO:teuthology.orchestra.run.mira079.stderr:gzip: /var/log/ceph/valgrind/*.gz: No such file or directory 2014-07-01 17:22:03,843.843 INFO:teuthology.orchestra.run.mira028:Running: "sudo zgrep '' /var/log/ceph/valgrind/* /dev/null | sort | uniq" 2014-07-01 17:22:03,894.894 INFO:teuthology.orchestra.run.mira043.stderr:gzip: /var/log/ceph/valgrind/*.gz: No such file or directory 2014-07-01 17:22:04,290.290 INFO:teuthology.orchestra.run.mira028.stderr:gzip: /var/log/ceph/valgrind/*.gz: No such file or directory 2014-07-01 17:22:04,305.305 INFO:teuthology.task.ceph:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/kcon_most... 2014-07-01 17:22:04,305.305 INFO:teuthology.orchestra.run.mira028:Running: 'sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/kcon_most' 2014-07-01 17:22:04,776.776 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/kcon_most' 2014-07-01 17:22:05,113.113 INFO:teuthology.orchestra.run.mira079:Running: 'sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/kcon_most' 2014-07-01 17:22:05,464.464 DEBUG:teuthology.run_tasks:Unwinding manager install 2014-07-01 17:22:05,465.465 INFO:teuthology.orchestra.run.mira079:Running: 'sudo lsb_release -is' 2014-07-01 17:22:06,103.103 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:06,103.103 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 17:22:06,105.105 INFO:teuthology.task.install:Removing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg on Debian system. 2014-07-01 17:22:06,105.105 INFO:teuthology.orchestra.run.mira079:Running: 'for d in ceph ceph-dbg ceph-mds ceph-mds-dbg ceph-common ceph-common-dbg ceph-fuse ceph-fuse-dbg ceph-test ceph-test-dbg radosgw radosgw-dbg python-ceph libcephfs1 libcephfs1-dbg ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done' 2014-07-01 17:22:06,526.526 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:06,527.527 INFO:teuthology.orchestra.run.mira028:Running: 'sudo lsb_release -is' 2014-07-01 17:22:06,529.529 INFO:teuthology.task.install:Removing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg on Debian system. 2014-07-01 17:22:06,530.530 INFO:teuthology.orchestra.run.mira043:Running: 'for d in ceph ceph-dbg ceph-mds ceph-mds-dbg ceph-common ceph-common-dbg ceph-fuse ceph-fuse-dbg ceph-test ceph-test-dbg radosgw radosgw-dbg python-ceph libcephfs1 libcephfs1-dbg ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done' 2014-07-01 17:22:06,895.895 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:06,994.994 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:06,995.995 INFO:teuthology.task.install:Removing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg on Debian system. 2014-07-01 17:22:06,996.996 INFO:teuthology.orchestra.run.mira028:Running: 'for d in ceph ceph-dbg ceph-mds ceph-mds-dbg ceph-common ceph-common-dbg ceph-fuse ceph-fuse-dbg ceph-test ceph-test-dbg radosgw radosgw-dbg python-ceph libcephfs1 libcephfs1-dbg ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done' 2014-07-01 17:22:07,054.054 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:07,314.314 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:07,316.316 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:07,620.620 INFO:teuthology.orchestra.run.mira043.stdout:The following packages will be REMOVED: 2014-07-01 17:22:07,621.621 INFO:teuthology.orchestra.run.mira043.stdout: ceph* ceph-dbg* ceph-mds* ceph-mds-dbg* 2014-07-01 17:22:07,778.778 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:07,784.784 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 4 to remove and 73 not upgraded. 2014-07-01 17:22:07,784.784 INFO:teuthology.orchestra.run.mira043.stdout:After this operation, 576 MB disk space will be freed. 2014-07-01 17:22:07,853.853 INFO:teuthology.orchestra.run.mira043.stdout:(Reading database ... 80487 files and directories currently installed.) 2014-07-01 17:22:07,865.865 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-mds-dbg ... 2014-07-01 17:22:07,908.908 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-mds ... 2014-07-01 17:22:07,949.949 INFO:teuthology.orchestra.run.mira043.stdout:ceph-mds-all stop/waiting 2014-07-01 17:22:07,975.975 INFO:teuthology.orchestra.run.mira043.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:07,976.976 INFO:teuthology.orchestra.run.mira043.stdout:Purging configuration files for ceph-mds ... 2014-07-01 17:22:08,016.016 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-dbg ... 2014-07-01 17:22:08,113.113 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph ... 2014-07-01 17:22:08,125.125 INFO:teuthology.orchestra.run.mira043.stdout:ceph-all stop/waiting 2014-07-01 17:22:08,158.158 INFO:teuthology.orchestra.run.mira043.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:08,212.212 INFO:teuthology.orchestra.run.mira043.stdout:Purging configuration files for ceph ... 2014-07-01 17:22:08,239.239 INFO:teuthology.orchestra.run.mira043.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/tmp' not empty so not removed. 2014-07-01 17:22:08,276.276 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:08,277.277 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for man-db ... 2014-07-01 17:22:08,495.495 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:08,518.518 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:08,568.568 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:08,571.571 INFO:teuthology.orchestra.run.mira043.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:08,770.770 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:08,859.859 INFO:teuthology.orchestra.run.mira079.stdout:The following packages will be REMOVED: 2014-07-01 17:22:08,859.859 INFO:teuthology.orchestra.run.mira079.stdout: ceph* ceph-dbg* ceph-mds* ceph-mds-dbg* 2014-07-01 17:22:09,034.034 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:09,034.034 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:09,271.271 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-dbg is not installed, so not removed 2014-07-01 17:22:09,271.271 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:09,294.294 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:09,325.325 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:09,340.340 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:09,555.555 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:09,556.556 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:09,658.658 INFO:teuthology.orchestra.run.mira028.stdout:The following packages will be REMOVED: 2014-07-01 17:22:09,658.658 INFO:teuthology.orchestra.run.mira028.stdout: ceph* ceph-dbg* ceph-mds* ceph-mds-dbg* 2014-07-01 17:22:09,782.782 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 4 to remove and 141 not upgraded. 2014-07-01 17:22:09,782.782 INFO:teuthology.orchestra.run.mira079.stdout:After this operation, 651 MB disk space will be freed. 2014-07-01 17:22:09,790.790 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-mds is not installed, so not removed 2014-07-01 17:22:09,791.791 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:09,817.817 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:09,848.848 INFO:teuthology.orchestra.run.mira079.stdout:(Reading database ... 77707 files and directories currently installed.) 2014-07-01 17:22:09,852.852 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-mds-dbg ... 2014-07-01 17:22:09,873.873 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-mds ... 2014-07-01 17:22:09,945.945 INFO:teuthology.orchestra.run.mira079.stdout:ceph-mds-all stop/waiting 2014-07-01 17:22:10,080.080 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:10,082.082 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:10,202.202 INFO:teuthology.orchestra.run.mira079.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:10,217.217 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for ceph-mds ... 2014-07-01 17:22:10,232.232 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-dbg ... 2014-07-01 17:22:10,259.259 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph ... 2014-07-01 17:22:10,265.265 INFO:teuthology.orchestra.run.mira079.stdout:ceph-all stop/waiting 2014-07-01 17:22:10,289.289 INFO:teuthology.orchestra.run.mira079.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:10,316.316 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-mds-dbg is not installed, so not removed 2014-07-01 17:22:10,316.316 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:10,339.339 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:10,342.342 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for ceph ... 2014-07-01 17:22:10,360.360 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/mds' not empty so not removed. 2014-07-01 17:22:10,361.361 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/osd' not empty so not removed. 2014-07-01 17:22:10,361.361 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/mon' not empty so not removed. 2014-07-01 17:22:10,361.361 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/tmp' not empty so not removed. 2014-07-01 17:22:10,362.362 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:10,364.364 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for man-db ... 2014-07-01 17:22:10,547.547 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 4 to remove and 43 not upgraded. 2014-07-01 17:22:10,547.547 INFO:teuthology.orchestra.run.mira028.stdout:After this operation, 651 MB disk space will be freed. 2014-07-01 17:22:10,599.599 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:10,601.601 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:10,794.794 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:10,795.795 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:10,795.795 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:10,947.947 INFO:teuthology.orchestra.run.mira043.stdout:The following packages will be REMOVED: 2014-07-01 17:22:10,947.947 INFO:teuthology.orchestra.run.mira043.stdout: ceph-common* ceph-common-dbg* ceph-test* ceph-test-dbg* radosgw* 2014-07-01 17:22:10,947.947 INFO:teuthology.orchestra.run.mira043.stdout: radosgw-dbg* 2014-07-01 17:22:10,948.948 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:10,951.951 INFO:teuthology.orchestra.run.mira079.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:11,110.110 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 6 to remove and 73 not upgraded. 2014-07-01 17:22:11,110.110 INFO:teuthology.orchestra.run.mira043.stdout:After this operation, 1,845 MB disk space will be freed. 2014-07-01 17:22:11,132.132 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:11,214.214 INFO:teuthology.orchestra.run.mira043.stdout:(Reading database ... 80300 files and directories currently installed.) 2014-07-01 17:22:11,217.217 INFO:teuthology.orchestra.run.mira043.stdout:Removing radosgw-dbg ... 2014-07-01 17:22:11,252.252 INFO:teuthology.orchestra.run.mira043.stdout:Removing radosgw ... 2014-07-01 17:22:11,273.273 INFO:teuthology.orchestra.run.mira043.stderr:stop: Unknown instance: 2014-07-01 17:22:11,330.330 INFO:teuthology.orchestra.run.mira043.stdout:No /usr/bin/radosgw found running; none killed. 2014-07-01 17:22:11,330.330 INFO:teuthology.orchestra.run.mira043.stdout:Purging configuration files for radosgw ... 2014-07-01 17:22:11,378.378 INFO:teuthology.orchestra.run.mira043.stderr:dpkg: warning: while removing radosgw, directory '/var/lib/ceph' not empty so not removed. 2014-07-01 17:22:11,378.378 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-test-dbg ... 2014-07-01 17:22:11,391.391 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:11,393.393 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:11,626.626 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-dbg is not installed, so not removed 2014-07-01 17:22:11,627.627 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:11,641.641 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:11,702.702 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-test ... 2014-07-01 17:22:11,755.755 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-common-dbg ... 2014-07-01 17:22:11,849.849 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-common ... 2014-07-01 17:22:11,874.874 INFO:teuthology.orchestra.run.mira043.stdout:Purging configuration files for ceph-common ... 2014-07-01 17:22:11,897.897 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:11,897.897 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:11,900.900 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:11,901.901 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for man-db ... 2014-07-01 17:22:12,131.131 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-mds is not installed, so not removed 2014-07-01 17:22:12,132.132 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:12,145.145 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:12,402.402 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:12,404.404 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:12,421.421 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:12,636.636 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-mds-dbg is not installed, so not removed 2014-07-01 17:22:12,636.636 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:12,652.652 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:12,682.682 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:12,684.684 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:12,687.687 INFO:teuthology.orchestra.run.mira028.stdout:(Reading database ... 79791 files and directories currently installed.) 2014-07-01 17:22:12,692.692 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-mds-dbg ... 2014-07-01 17:22:12,719.719 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-mds ... 2014-07-01 17:22:12,787.787 INFO:teuthology.orchestra.run.mira028.stdout:ceph-mds-all stop/waiting 2014-07-01 17:22:12,874.874 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-common-dbg is not installed, so not removed 2014-07-01 17:22:12,875.875 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:12,875.875 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:12,875.875 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:12,910.910 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:12,911.911 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:12,920.920 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:12,944.944 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:13,099.099 INFO:teuthology.orchestra.run.mira028.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:13,100.100 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:13,100.100 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:13,100.100 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:13,114.114 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for ceph-mds ... 2014-07-01 17:22:13,137.137 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-dbg ... 2014-07-01 17:22:13,167.167 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph ... 2014-07-01 17:22:13,204.204 INFO:teuthology.orchestra.run.mira028.stdout:ceph-all stop/waiting 2014-07-01 17:22:13,205.205 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:13,207.207 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:13,227.227 INFO:teuthology.orchestra.run.mira028.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-07-01 17:22:13,249.249 INFO:teuthology.orchestra.run.mira079.stdout:The following packages will be REMOVED: 2014-07-01 17:22:13,250.250 INFO:teuthology.orchestra.run.mira079.stdout: ceph-common* ceph-common-dbg* ceph-test* ceph-test-dbg* radosgw* 2014-07-01 17:22:13,250.250 INFO:teuthology.orchestra.run.mira079.stdout: radosgw-dbg* 2014-07-01 17:22:13,305.305 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for ceph ... 2014-07-01 17:22:13,400.400 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:13,401.401 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:13,401.401 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:13,410.410 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 6 to remove and 141 not upgraded. 2014-07-01 17:22:13,410.410 INFO:teuthology.orchestra.run.mira079.stdout:After this operation, 1,919 MB disk space will be freed. 2014-07-01 17:22:13,470.470 INFO:teuthology.orchestra.run.mira043.stdout:The following packages will be REMOVED: 2014-07-01 17:22:13,470.470 INFO:teuthology.orchestra.run.mira043.stdout: ceph-fuse* ceph-fuse-dbg* 2014-07-01 17:22:13,483.483 INFO:teuthology.orchestra.run.mira079.stdout:(Reading database ... 77516 files and directories currently installed.) 2014-07-01 17:22:13,486.486 INFO:teuthology.orchestra.run.mira079.stdout:Removing radosgw-dbg ... 2014-07-01 17:22:13,502.502 INFO:teuthology.orchestra.run.mira079.stdout:Removing radosgw ... 2014-07-01 17:22:13,524.524 INFO:teuthology.orchestra.run.mira079.stderr:stop: Unknown instance: 2014-07-01 17:22:13,564.564 INFO:teuthology.orchestra.run.mira079.stdout:No /usr/bin/radosgw found running; none killed. 2014-07-01 17:22:13,576.576 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/osd' not empty so not removed. 2014-07-01 17:22:13,576.576 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/mon' not empty so not removed. 2014-07-01 17:22:13,576.576 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/tmp' not empty so not removed. 2014-07-01 17:22:13,577.577 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:13,581.581 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for man-db ... 2014-07-01 17:22:13,583.583 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for radosgw ... 2014-07-01 17:22:13,608.608 INFO:teuthology.orchestra.run.mira079.stderr:dpkg: warning: while removing radosgw, directory '/var/lib/ceph' not empty so not removed. 2014-07-01 17:22:13,619.619 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-test-dbg ... 2014-07-01 17:22:13,641.641 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 2 to remove and 73 not upgraded. 2014-07-01 17:22:13,641.641 INFO:teuthology.orchestra.run.mira043.stdout:After this operation, 81.1 MB disk space will be freed. 2014-07-01 17:22:13,676.676 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-test ... 2014-07-01 17:22:13,698.698 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-common-dbg ... 2014-07-01 17:22:13,721.721 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-common ... 2014-07-01 17:22:13,736.736 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for ceph-common ... 2014-07-01 17:22:13,740.740 INFO:teuthology.orchestra.run.mira043.stdout:(Reading database ... 80060 files and directories currently installed.) 2014-07-01 17:22:13,742.742 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-fuse-dbg ... 2014-07-01 17:22:13,780.780 INFO:teuthology.orchestra.run.mira043.stdout:Removing ceph-fuse ... 2014-07-01 17:22:13,805.805 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for man-db ... 2014-07-01 17:22:14,051.051 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:14,054.054 INFO:teuthology.orchestra.run.mira028.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:14,220.220 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:14,340.340 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:14,480.480 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:14,482.482 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:14,602.602 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:14,604.604 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:14,670.670 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-dbg is not installed, so not removed 2014-07-01 17:22:14,670.670 INFO:teuthology.orchestra.run.mira028.stdout:The following package was automatically installed and is no longer required: 2014-07-01 17:22:14,670.670 INFO:teuthology.orchestra.run.mira028.stdout: ceph-fs-common 2014-07-01 17:22:14,670.670 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:14,717.717 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:14,739.739 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:14,792.792 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-fuse-dbg is not installed, so not removed 2014-07-01 17:22:14,793.793 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:14,793.793 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:14,793.793 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:14,839.839 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:14,851.851 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:14,853.853 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for man-db ... 2014-07-01 17:22:14,867.867 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:14,999.999 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:15,000.000 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:15,125.125 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:15,127.127 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:15,188.188 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-mds is not installed, so not removed 2014-07-01 17:22:15,188.188 INFO:teuthology.orchestra.run.mira028.stdout:The following package was automatically installed and is no longer required: 2014-07-01 17:22:15,188.188 INFO:teuthology.orchestra.run.mira028.stdout: ceph-fs-common 2014-07-01 17:22:15,189.189 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:15,233.233 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:15,252.252 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:15,307.307 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:15,316.316 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-test is not installed, so not removed 2014-07-01 17:22:15,316.316 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:15,316.316 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:15,316.316 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:15,367.367 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:15,386.386 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:15,514.514 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:15,515.515 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:15,566.566 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:15,567.567 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:15,646.646 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:15,648.648 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:15,703.703 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-mds-dbg is not installed, so not removed 2014-07-01 17:22:15,703.703 INFO:teuthology.orchestra.run.mira028.stdout:The following package was automatically installed and is no longer required: 2014-07-01 17:22:15,704.704 INFO:teuthology.orchestra.run.mira028.stdout: ceph-fs-common 2014-07-01 17:22:15,704.704 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:15,750.750 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:15,755.755 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-common-dbg is not installed, so not removed 2014-07-01 17:22:15,755.755 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:15,755.755 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:15,756.756 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:15,772.772 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:15,800.800 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:15,822.822 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:15,838.838 INFO:teuthology.orchestra.run.mira043.stdout:Package ceph-test-dbg is not installed, so not removed 2014-07-01 17:22:15,838.838 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:15,838.838 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:15,838.838 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:15,882.882 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:15,900.900 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:16,032.032 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:16,034.034 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:16,078.078 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:16,080.080 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:16,157.157 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:16,158.158 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:16,226.226 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:16,226.226 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:16,226.226 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:16,226.226 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:16,271.271 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:16,271.271 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:16,272.272 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:16,341.341 INFO:teuthology.orchestra.run.mira079.stdout:The following packages will be REMOVED: 2014-07-01 17:22:16,341.341 INFO:teuthology.orchestra.run.mira079.stdout: ceph-fuse* ceph-fuse-dbg* 2014-07-01 17:22:16,348.348 INFO:teuthology.orchestra.run.mira043.stdout:Package radosgw is not installed, so not removed 2014-07-01 17:22:16,348.348 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:16,348.348 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:16,348.348 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:16,377.377 INFO:teuthology.orchestra.run.mira028.stdout:The following packages will be REMOVED: 2014-07-01 17:22:16,378.378 INFO:teuthology.orchestra.run.mira028.stdout: ceph-common* ceph-common-dbg* ceph-test* ceph-test-dbg* radosgw* 2014-07-01 17:22:16,378.378 INFO:teuthology.orchestra.run.mira028.stdout: radosgw-dbg* 2014-07-01 17:22:16,392.392 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:16,417.417 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:16,498.498 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 2 to remove and 141 not upgraded. 2014-07-01 17:22:16,498.498 INFO:teuthology.orchestra.run.mira079.stdout:After this operation, 82.6 MB disk space will be freed. 2014-07-01 17:22:16,537.537 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 6 to remove and 43 not upgraded. 2014-07-01 17:22:16,537.537 INFO:teuthology.orchestra.run.mira028.stdout:After this operation, 1,919 MB disk space will be freed. 2014-07-01 17:22:16,569.569 INFO:teuthology.orchestra.run.mira079.stdout:(Reading database ... 77262 files and directories currently installed.) 2014-07-01 17:22:16,571.571 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-fuse-dbg ... 2014-07-01 17:22:16,588.588 INFO:teuthology.orchestra.run.mira079.stdout:Removing ceph-fuse ... 2014-07-01 17:22:16,598.598 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for man-db ... 2014-07-01 17:22:16,622.622 INFO:teuthology.orchestra.run.mira028.stdout:(Reading database ... 79600 files and directories currently installed.) 2014-07-01 17:22:16,625.625 INFO:teuthology.orchestra.run.mira028.stdout:Removing radosgw-dbg ... 2014-07-01 17:22:16,649.649 INFO:teuthology.orchestra.run.mira028.stdout:Removing radosgw ... 2014-07-01 17:22:16,670.670 INFO:teuthology.orchestra.run.mira028.stderr:stop: Unknown instance: 2014-07-01 17:22:16,681.681 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:16,683.683 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:16,732.732 INFO:teuthology.orchestra.run.mira028.stdout:No /usr/bin/radosgw found running; none killed. 2014-07-01 17:22:16,751.751 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for radosgw ... 2014-07-01 17:22:16,776.776 INFO:teuthology.orchestra.run.mira028.stderr:dpkg: warning: while removing radosgw, directory '/var/lib/ceph' not empty so not removed. 2014-07-01 17:22:16,787.787 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-test-dbg ... 2014-07-01 17:22:16,841.841 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-test ... 2014-07-01 17:22:16,872.872 INFO:teuthology.orchestra.run.mira043.stdout:Package radosgw-dbg is not installed, so not removed 2014-07-01 17:22:16,872.872 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:16,872.872 INFO:teuthology.orchestra.run.mira043.stdout: libboost-program-options1.46.1 libleveldb1 xml2 libsnappy1 2014-07-01 17:22:16,873.873 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:16,874.874 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-common-dbg ... 2014-07-01 17:22:16,905.905 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-common ... 2014-07-01 17:22:16,916.916 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:16,920.920 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for ceph-common ... 2014-07-01 17:22:16,939.939 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:17,025.025 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:17,199.199 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:17,201.201 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:17,282.282 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:17,283.283 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:17,390.390 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:17,390.390 INFO:teuthology.orchestra.run.mira043.stdout: python-flask libboost-program-options1.46.1 libleveldb1 xml2 2014-07-01 17:22:17,391.391 INFO:teuthology.orchestra.run.mira043.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug libsnappy1 2014-07-01 17:22:17,391.391 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:17,438.438 INFO:teuthology.orchestra.run.mira043.stdout:The following packages will be REMOVED: 2014-07-01 17:22:17,439.439 INFO:teuthology.orchestra.run.mira043.stdout: python-ceph* 2014-07-01 17:22:17,471.471 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-fuse-dbg is not installed, so not removed 2014-07-01 17:22:17,471.471 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:17,471.471 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:17,471.471 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:17,516.516 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:17,540.540 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:17,600.600 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 1 to remove and 73 not upgraded. 2014-07-01 17:22:17,601.601 INFO:teuthology.orchestra.run.mira043.stdout:After this operation, 172 kB disk space will be freed. 2014-07-01 17:22:17,686.686 INFO:teuthology.orchestra.run.mira043.stdout:(Reading database ... 80049 files and directories currently installed.) 2014-07-01 17:22:17,689.689 INFO:teuthology.orchestra.run.mira043.stdout:Removing python-ceph ... 2014-07-01 17:22:17,799.799 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:17,800.800 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:17,966.966 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:17,988.988 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-test is not installed, so not removed 2014-07-01 17:22:17,988.988 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:17,989.989 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:17,989.989 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:18,032.032 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:18,047.047 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:18,226.226 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:18,227.227 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:18,308.308 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for ureadahead ... 2014-07-01 17:22:18,313.313 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:18,313.313 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for man-db ... 2014-07-01 17:22:18,314.314 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:18,423.423 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:18,424.424 INFO:teuthology.orchestra.run.mira043.stdout: python-flask libboost-program-options1.46.1 libleveldb1 xml2 2014-07-01 17:22:18,424.424 INFO:teuthology.orchestra.run.mira043.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug libsnappy1 2014-07-01 17:22:18,424.424 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:18,495.495 INFO:teuthology.orchestra.run.mira043.stdout:The following packages will be REMOVED: 2014-07-01 17:22:18,495.495 INFO:teuthology.orchestra.run.mira043.stdout: libcephfs1* libcephfs1-dbg* 2014-07-01 17:22:18,502.502 INFO:teuthology.orchestra.run.mira079.stdout:Package ceph-test-dbg is not installed, so not removed 2014-07-01 17:22:18,502.502 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:18,502.502 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:18,502.502 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:18,547.547 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:18,571.571 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:18,657.657 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 2 to remove and 73 not upgraded. 2014-07-01 17:22:18,657.657 INFO:teuthology.orchestra.run.mira043.stdout:After this operation, 91.2 MB disk space will be freed. 2014-07-01 17:22:18,749.749 INFO:teuthology.orchestra.run.mira043.stdout:(Reading database ... 80036 files and directories currently installed.) 2014-07-01 17:22:18,751.751 INFO:teuthology.orchestra.run.mira043.stdout:Removing libcephfs1-dbg ... 2014-07-01 17:22:18,779.779 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:18,794.794 INFO:teuthology.orchestra.run.mira043.stdout:Removing libcephfs1 ... 2014-07-01 17:22:18,828.828 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:18,830.830 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:18,873.873 INFO:teuthology.orchestra.run.mira043.stdout:Purging configuration files for libcephfs1 ... 2014-07-01 17:22:18,874.874 INFO:teuthology.orchestra.run.mira043.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:18,874.874 INFO:teuthology.orchestra.run.mira043.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:19,017.017 INFO:teuthology.orchestra.run.mira079.stdout:Package radosgw is not installed, so not removed 2014-07-01 17:22:19,017.017 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:19,018.018 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:19,018.018 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:19,033.033 INFO:teuthology.orchestra.run.mira043.stdout:Reading package lists... 2014-07-01 17:22:19,037.037 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:19,039.039 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:19,065.065 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:19,085.085 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:19,227.227 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-common-dbg is not installed, so not removed 2014-07-01 17:22:19,228.228 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:19,228.228 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:19,228.228 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:19,228.228 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:19,273.273 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:19,292.292 INFO:teuthology.orchestra.run.mira043.stdout:Building dependency tree... 2014-07-01 17:22:19,294.294 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:19,294.294 INFO:teuthology.orchestra.run.mira043.stdout:Reading state information... 2014-07-01 17:22:19,345.345 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:19,346.346 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:19,488.488 INFO:teuthology.orchestra.run.mira043.stdout:Package libcephfs1-dbg is not installed, so not removed 2014-07-01 17:22:19,488.488 INFO:teuthology.orchestra.run.mira043.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:19,488.488 INFO:teuthology.orchestra.run.mira043.stdout: python-flask libboost-program-options1.46.1 libleveldb1 xml2 2014-07-01 17:22:19,488.488 INFO:teuthology.orchestra.run.mira043.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug libsnappy1 2014-07-01 17:22:19,488.488 INFO:teuthology.orchestra.run.mira043.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:19,534.534 INFO:teuthology.orchestra.run.mira079.stdout:Package radosgw-dbg is not installed, so not removed 2014-07-01 17:22:19,534.534 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:19,534.534 INFO:teuthology.orchestra.run.mira079.stdout: libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:19,534.534 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:19,535.535 INFO:teuthology.orchestra.run.mira043.stdout:0 upgraded, 0 newly installed, 0 to remove and 73 not upgraded. 2014-07-01 17:22:19,537.537 INFO:teuthology.orchestra.run.mira043:Running: "dpkg -l | grep '^.HR' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq" 2014-07-01 17:22:19,555.555 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:19,557.557 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:19,580.580 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:19,603.603 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:19,748.748 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:19,748.748 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:19,748.748 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:19,748.748 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:19,818.818 INFO:teuthology.orchestra.run.mira028.stdout:The following packages will be REMOVED: 2014-07-01 17:22:19,818.818 INFO:teuthology.orchestra.run.mira028.stdout: ceph-fuse* ceph-fuse-dbg* 2014-07-01 17:22:19,861.861 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:19,862.862 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:19,972.972 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 2 to remove and 43 not upgraded. 2014-07-01 17:22:19,972.972 INFO:teuthology.orchestra.run.mira028.stdout:After this operation, 82.6 MB disk space will be freed. 2014-07-01 17:22:20,047.047 INFO:teuthology.orchestra.run.mira028.stdout:(Reading database ... 79346 files and directories currently installed.) 2014-07-01 17:22:20,048.048 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:20,048.048 INFO:teuthology.orchestra.run.mira079.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:20,048.048 INFO:teuthology.orchestra.run.mira079.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug 2014-07-01 17:22:20,048.048 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:20,049.049 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-fuse-dbg ... 2014-07-01 17:22:20,069.069 INFO:teuthology.orchestra.run.mira028.stdout:Removing ceph-fuse ... 2014-07-01 17:22:20,083.083 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for man-db ... 2014-07-01 17:22:20,096.096 INFO:teuthology.orchestra.run.mira079.stdout:The following packages will be REMOVED: 2014-07-01 17:22:20,097.097 INFO:teuthology.orchestra.run.mira079.stdout: python-ceph* 2014-07-01 17:22:20,195.195 INFO:teuthology.orchestra.run.mira043:Running: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove' 2014-07-01 17:22:20,257.257 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 1 to remove and 141 not upgraded. 2014-07-01 17:22:20,257.257 INFO:teuthology.orchestra.run.mira079.stdout:After this operation, 834 kB disk space will be freed. 2014-07-01 17:22:20,341.341 INFO:teuthology.orchestra.run.mira079.stdout:(Reading database ... 77249 files and directories currently installed.) 2014-07-01 17:22:20,343.343 INFO:teuthology.orchestra.run.mira079.stdout:Removing python-ceph ... 2014-07-01 17:22:20,530.530 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:20,587.587 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:20,790.790 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:20,792.792 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:20,845.845 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:20,847.847 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:20,982.982 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-fuse-dbg is not installed, so not removed 2014-07-01 17:22:20,982.982 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:20,982.982 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:20,982.982 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:20,982.982 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:21,027.027 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:21,038.038 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:21,038.038 INFO:teuthology.orchestra.run.mira079.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:21,038.038 INFO:teuthology.orchestra.run.mira079.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug 2014-07-01 17:22:21,038.038 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:21,049.049 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:21,148.148 INFO:teuthology.orchestra.run.mira079.stdout:The following packages will be REMOVED: 2014-07-01 17:22:21,148.148 INFO:teuthology.orchestra.run.mira079.stdout: libcephfs-java* libcephfs-jni* libcephfs1* libcephfs1-dbg* 2014-07-01 17:22:21,310.310 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 4 to remove and 141 not upgraded. 2014-07-01 17:22:21,311.311 INFO:teuthology.orchestra.run.mira079.stdout:After this operation, 95.0 MB disk space will be freed. 2014-07-01 17:22:21,315.315 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:21,318.318 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:21,383.383 INFO:teuthology.orchestra.run.mira079.stdout:(Reading database ... 77235 files and directories currently installed.) 2014-07-01 17:22:21,385.385 INFO:teuthology.orchestra.run.mira079.stdout:Removing libcephfs-java ... 2014-07-01 17:22:21,401.401 INFO:teuthology.orchestra.run.mira079.stdout:Removing libcephfs-jni ... 2014-07-01 17:22:21,420.420 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for libcephfs-jni ... 2014-07-01 17:22:21,436.436 INFO:teuthology.orchestra.run.mira079.stdout:Removing libcephfs1-dbg ... 2014-07-01 17:22:21,454.454 INFO:teuthology.orchestra.run.mira079.stdout:Removing libcephfs1 ... 2014-07-01 17:22:21,469.469 INFO:teuthology.orchestra.run.mira079.stdout:Purging configuration files for libcephfs1 ... 2014-07-01 17:22:21,478.478 INFO:teuthology.orchestra.run.mira079.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:21,479.479 INFO:teuthology.orchestra.run.mira079.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:21,504.504 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-test is not installed, so not removed 2014-07-01 17:22:21,504.504 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:21,504.504 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:21,504.504 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:21,504.504 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:21,549.549 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:21,565.565 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:21,648.648 INFO:teuthology.orchestra.run.mira079.stdout:Reading package lists... 2014-07-01 17:22:21,822.822 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:21,824.824 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:21,907.907 INFO:teuthology.orchestra.run.mira079.stdout:Building dependency tree... 2014-07-01 17:22:21,909.909 INFO:teuthology.orchestra.run.mira079.stdout:Reading state information... 2014-07-01 17:22:22,012.012 INFO:teuthology.orchestra.run.mira028.stdout:Package ceph-test-dbg is not installed, so not removed 2014-07-01 17:22:22,012.012 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:22,012.012 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:22,012.012 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:22,012.012 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:22,058.058 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:22,081.081 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:22,097.097 INFO:teuthology.orchestra.run.mira079.stdout:Package libcephfs1-dbg is not installed, so not removed 2014-07-01 17:22:22,097.097 INFO:teuthology.orchestra.run.mira079.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:22,098.098 INFO:teuthology.orchestra.run.mira079.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 xml2 2014-07-01 17:22:22,098.098 INFO:teuthology.orchestra.run.mira079.stdout: python-markupsafe python-jinja2 libjs-jquery python-werkzeug 2014-07-01 17:22:22,098.098 INFO:teuthology.orchestra.run.mira079.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:22,143.143 INFO:teuthology.orchestra.run.mira079.stdout:0 upgraded, 0 newly installed, 0 to remove and 141 not upgraded. 2014-07-01 17:22:22,145.145 INFO:teuthology.orchestra.run.mira079:Running: "dpkg -l | grep '^.HR' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq" 2014-07-01 17:22:22,341.341 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:22,342.342 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:22,529.529 INFO:teuthology.orchestra.run.mira028.stdout:Package radosgw is not installed, so not removed 2014-07-01 17:22:22,529.529 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:22,530.530 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:22,530.530 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:22,530.530 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:22,572.572 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:22,595.595 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:22,678.678 INFO:teuthology.orchestra.run.mira079:Running: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove' 2014-07-01 17:22:22,853.853 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:22,856.856 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:22,915.915 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:23,045.045 INFO:teuthology.orchestra.run.mira028.stdout:Package radosgw-dbg is not installed, so not removed 2014-07-01 17:22:23,045.045 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:23,045.045 INFO:teuthology.orchestra.run.mira028.stdout: libradosstriper1 libboost-program-options1.46.1 ceph-fs-common python-six 2014-07-01 17:22:23,045.045 INFO:teuthology.orchestra.run.mira028.stdout: libleveldb1 python-urllib3 xml2 python-requests libsnappy1 2014-07-01 17:22:23,046.046 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:23,088.088 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:23,110.110 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:23,368.368 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:23,369.369 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:23,559.559 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:23,560.560 INFO:teuthology.orchestra.run.mira028.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 ceph-fs-common 2014-07-01 17:22:23,560.560 INFO:teuthology.orchestra.run.mira028.stdout: python-six libleveldb1 python-urllib3 xml2 python-markupsafe python-jinja2 2014-07-01 17:22:23,560.560 INFO:teuthology.orchestra.run.mira028.stdout: libjs-jquery python-werkzeug python-requests libsnappy1 2014-07-01 17:22:23,560.560 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:23,607.607 INFO:teuthology.orchestra.run.mira028.stdout:The following packages will be REMOVED: 2014-07-01 17:22:23,607.607 INFO:teuthology.orchestra.run.mira028.stdout: python-ceph* 2014-07-01 17:22:23,761.761 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 1 to remove and 43 not upgraded. 2014-07-01 17:22:23,761.761 INFO:teuthology.orchestra.run.mira028.stdout:After this operation, 834 kB disk space will be freed. 2014-07-01 17:22:23,842.842 INFO:teuthology.orchestra.run.mira028.stdout:(Reading database ... 79333 files and directories currently installed.) 2014-07-01 17:22:23,844.844 INFO:teuthology.orchestra.run.mira028.stdout:Removing python-ceph ... 2014-07-01 17:22:24,091.091 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:24,350.350 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:24,352.352 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:24,543.543 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:24,544.544 INFO:teuthology.orchestra.run.mira028.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 ceph-fs-common 2014-07-01 17:22:24,544.544 INFO:teuthology.orchestra.run.mira028.stdout: python-six libleveldb1 python-urllib3 xml2 python-markupsafe python-jinja2 2014-07-01 17:22:24,544.544 INFO:teuthology.orchestra.run.mira028.stdout: libjs-jquery python-werkzeug python-requests libsnappy1 2014-07-01 17:22:24,544.544 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:24,653.653 INFO:teuthology.orchestra.run.mira028.stdout:The following packages will be REMOVED: 2014-07-01 17:22:24,654.654 INFO:teuthology.orchestra.run.mira028.stdout: libcephfs-java* libcephfs-jni* libcephfs1* libcephfs1-dbg* 2014-07-01 17:22:24,806.806 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 4 to remove and 43 not upgraded. 2014-07-01 17:22:24,806.806 INFO:teuthology.orchestra.run.mira028.stdout:After this operation, 95.0 MB disk space will be freed. 2014-07-01 17:22:24,892.892 INFO:teuthology.orchestra.run.mira028.stdout:(Reading database ... 79319 files and directories currently installed.) 2014-07-01 17:22:24,895.895 INFO:teuthology.orchestra.run.mira028.stdout:Removing libcephfs-java ... 2014-07-01 17:22:24,911.911 INFO:teuthology.orchestra.run.mira028.stdout:Removing libcephfs-jni ... 2014-07-01 17:22:24,952.952 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for libcephfs-jni ... 2014-07-01 17:22:24,971.971 INFO:teuthology.orchestra.run.mira028.stdout:Removing libcephfs1-dbg ... 2014-07-01 17:22:24,988.988 INFO:teuthology.orchestra.run.mira028.stdout:Removing libcephfs1 ... 2014-07-01 17:22:25,006.006 INFO:teuthology.orchestra.run.mira028.stdout:Purging configuration files for libcephfs1 ... 2014-07-01 17:22:25,016.016 INFO:teuthology.orchestra.run.mira028.stdout:Processing triggers for libc-bin ... 2014-07-01 17:22:25,019.019 INFO:teuthology.orchestra.run.mira028.stdout:ldconfig deferred processing now taking place 2014-07-01 17:22:25,190.190 INFO:teuthology.orchestra.run.mira028.stdout:Reading package lists... 2014-07-01 17:22:25,244.244 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:25,446.446 INFO:teuthology.orchestra.run.mira028.stdout:Building dependency tree... 2014-07-01 17:22:25,448.448 INFO:teuthology.orchestra.run.mira028.stdout:Reading state information... 2014-07-01 17:22:25,634.634 INFO:teuthology.orchestra.run.mira028.stdout:Package libcephfs1-dbg is not installed, so not removed 2014-07-01 17:22:25,634.634 INFO:teuthology.orchestra.run.mira028.stdout:The following packages were automatically installed and are no longer required: 2014-07-01 17:22:25,634.634 INFO:teuthology.orchestra.run.mira028.stdout: python-flask libradosstriper1 libboost-program-options1.46.1 ceph-fs-common 2014-07-01 17:22:25,635.635 INFO:teuthology.orchestra.run.mira028.stdout: python-six libleveldb1 python-urllib3 xml2 python-markupsafe python-jinja2 2014-07-01 17:22:25,635.635 INFO:teuthology.orchestra.run.mira028.stdout: libjs-jquery python-werkzeug python-requests libsnappy1 2014-07-01 17:22:25,635.635 INFO:teuthology.orchestra.run.mira028.stdout:Use 'apt-get autoremove' to remove them. 2014-07-01 17:22:25,679.679 INFO:teuthology.orchestra.run.mira028.stdout:0 upgraded, 0 newly installed, 0 to remove and 43 not upgraded. 2014-07-01 17:22:25,681.681 INFO:teuthology.orchestra.run.mira028:Running: "dpkg -l | grep '^.HR' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq" 2014-07-01 17:22:26,215.215 INFO:teuthology.orchestra.run.mira028:Running: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove' 2014-07-01 17:22:29,482.482 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:29,482.482 INFO:teuthology.task.install:Removing ceph sources lists 2014-07-01 17:22:29,483.483 INFO:teuthology.orchestra.run.mira079:Running: 'sudo lsb_release -is' 2014-07-01 17:22:29,904.904 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:29,905.905 INFO:teuthology.orchestra.run.mira043:Running: 'sudo lsb_release -is' 2014-07-01 17:22:29,905.905 INFO:teuthology.orchestra.run.mira079:Running: 'sudo rm -f /etc/apt/sources.list.d/ceph.list && sudo apt-get update || true' 2014-07-01 17:22:29,906.906 INFO:teuthology.orchestra.run.mira079:Running: 'sudo rm -f /etc/apt/sources.list.d/calamari.list && sudo apt-get update || true' 2014-07-01 17:22:30,332.332 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:30,333.333 INFO:teuthology.orchestra.run.mira028:Running: 'sudo lsb_release -is' 2014-07-01 17:22:30,334.334 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -f /etc/apt/sources.list.d/ceph.list && sudo apt-get update || true' 2014-07-01 17:22:30,336.336 INFO:teuthology.orchestra.run.mira043:Running: 'sudo rm -f /etc/apt/sources.list.d/calamari.list && sudo apt-get update || true' 2014-07-01 17:22:30,715.715 INFO:teuthology.orchestra.run.mira079.stderr:E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable) 2014-07-01 17:22:30,715.715 INFO:teuthology.orchestra.run.mira079.stderr:E: Unable to lock directory /var/lib/apt/lists/ 2014-07-01 17:22:30,742.742 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-07-01 17:22:30,742.742 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:30,742.742 INFO:teuthology.orchestra.run.mira028:Running: 'sudo rm -f /etc/apt/sources.list.d/ceph.list && sudo apt-get update || true' 2014-07-01 17:22:30,743.743 INFO:teuthology.orchestra.run.mira028:Running: 'sudo rm -f /etc/apt/sources.list.d/calamari.list && sudo apt-get update || true' 2014-07-01 17:22:31,402.402 INFO:teuthology.orchestra.run.mira043.stderr:E: Could not get lock /var/lib/dpkg/lock - open (11: Resource temporarily unavailable) 2014-07-01 17:22:31,402.402 INFO:teuthology.orchestra.run.mira043.stderr:E: Unable to lock the administration directory (/var/lib/dpkg/), is another process using it? 2014-07-01 17:22:31,402.402 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:31,507.507 INFO:teuthology.orchestra.run.mira028.stderr:E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable) 2014-07-01 17:22:31,507.507 INFO:teuthology.orchestra.run.mira028.stderr:E: Unable to lock directory /var/lib/apt/lists/ 2014-07-01 17:22:31,507.507 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:33,388.388 INFO:teuthology.orchestra.run.mira043.stderr:E: Problem renaming the file /var/cache/apt/pkgcache.bin.3PuSPU to /var/cache/apt/pkgcache.bin - rename (2: No such file or directory) 2014-07-01 17:22:33,389.389 INFO:teuthology.orchestra.run.mira043.stderr:W: You may want to run apt-get update to correct these problems 2014-07-01 17:22:33,393.393 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:35,213.213 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:35,926.926 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:35,927.927 INFO:teuthology.task.install:Purging /var/lib/ceph on ubuntu@mira079.front.sepia.ceph.com 2014-07-01 17:22:35,928.928 INFO:teuthology.orchestra.run.mira079:Running: "sudo rm -rf --one-file-system -- /var/lib/ceph || true ; test -d /var/lib/ceph && sudo find /var/lib/ceph -mindepth 1 -maxdepth 2 -type d -exec umount '{}' ';' ; sudo rm -rf --one-file-system -- /var/lib/ceph" 2014-07-01 17:22:35,929.929 INFO:teuthology.task.install:Purging /var/lib/ceph on ubuntu@mira043.front.sepia.ceph.com 2014-07-01 17:22:35,930.930 INFO:teuthology.orchestra.run.mira043:Running: "sudo rm -rf --one-file-system -- /var/lib/ceph || true ; test -d /var/lib/ceph && sudo find /var/lib/ceph -mindepth 1 -maxdepth 2 -type d -exec umount '{}' ';' ; sudo rm -rf --one-file-system -- /var/lib/ceph" 2014-07-01 17:22:35,931.931 INFO:teuthology.task.install:Purging /var/lib/ceph on ubuntu@mira028.front.sepia.ceph.com 2014-07-01 17:22:35,931.931 INFO:teuthology.orchestra.run.mira028:Running: "sudo rm -rf --one-file-system -- /var/lib/ceph || true ; test -d /var/lib/ceph && sudo find /var/lib/ceph -mindepth 1 -maxdepth 2 -type d -exec umount '{}' ';' ; sudo rm -rf --one-file-system -- /var/lib/ceph" 2014-07-01 17:22:36,357.357 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:36,360.360 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:36,553.553 DEBUG:teuthology.parallel:result is None 2014-07-01 17:22:36,553.553 DEBUG:teuthology.run_tasks:Unwinding manager clock.check 2014-07-01 17:22:36,553.553 INFO:teuthology.task.clock:Checking final clock skew... 2014-07-01 17:22:36,554.554 INFO:teuthology.orchestra.run.mira079:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 17:22:37,020.020 INFO:teuthology.orchestra.run.mira079.stdout: remote local st poll reach delay offset disp 2014-07-01 17:22:37,020.020 INFO:teuthology.orchestra.run.mira079.stdout:======================================================================= 2014-07-01 17:22:37,020.020 INFO:teuthology.orchestra.run.mira079.stdout:*clock3.dreamhos 10.214.135.128 1 128 377 0.00043 -0.001056 0.06520 2014-07-01 17:22:37,021.021 INFO:teuthology.orchestra.run.mira043:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 17:22:37,418.418 INFO:teuthology.orchestra.run.mira043.stdout: remote local st poll reach delay offset disp 2014-07-01 17:22:37,419.419 INFO:teuthology.orchestra.run.mira043.stdout:======================================================================= 2014-07-01 17:22:37,419.419 INFO:teuthology.orchestra.run.mira043.stdout:*clock3.dreamhos 10.214.133.128 1 128 377 0.00044 -0.000158 0.08929 2014-07-01 17:22:37,419.419 INFO:teuthology.orchestra.run.mira028:Running: 'PATH=/usr/bin:/usr/sbin ntpdc -p' 2014-07-01 17:22:37,848.848 INFO:teuthology.orchestra.run.mira028.stdout: remote local st poll reach delay offset disp 2014-07-01 17:22:37,848.848 INFO:teuthology.orchestra.run.mira028.stdout:======================================================================= 2014-07-01 17:22:37,848.848 INFO:teuthology.orchestra.run.mira028.stdout:*clock3.dreamhos 10.214.132.120 1 128 377 0.00046 0.002705 0.10498 2014-07-01 17:22:37,849.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2014-07-01 17:22:37,849.849 INFO:teuthology.task.internal:Duration was 4417.487178 seconds 2014-07-01 17:22:37,849.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2014-07-01 17:22:37,849.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2014-07-01 17:22:37,849.849 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2014-07-01 17:22:37,849.849 INFO:teuthology.orchestra.run.mira028:Running: 'sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers' 2014-07-01 17:22:38,357.357 INFO:teuthology.orchestra.run.mira043:Running: 'sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers' 2014-07-01 17:22:38,746.746 INFO:teuthology.orchestra.run.mira079:Running: 'sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers' 2014-07-01 17:22:39,161.161 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2014-07-01 17:22:39,162.162 INFO:teuthology.orchestra.run.mira028:Running: 'sudo sysctl -w kernel.core_pattern=core && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump' 2014-07-01 17:22:39,532.532 INFO:teuthology.orchestra.run.mira043:Running: 'sudo sysctl -w kernel.core_pattern=core && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump' 2014-07-01 17:22:39,580.580 INFO:teuthology.orchestra.run.mira028.stdout:kernel.core_pattern = core 2014-07-01 17:22:39,903.903 INFO:teuthology.orchestra.run.mira079:Running: 'sudo sysctl -w kernel.core_pattern=core && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump' 2014-07-01 17:22:39,918.918 INFO:teuthology.orchestra.run.mira043.stdout:kernel.core_pattern = core 2014-07-01 17:22:40,338.338 INFO:teuthology.orchestra.run.mira079.stdout:kernel.core_pattern = core 2014-07-01 17:22:40,355.355 INFO:teuthology.orchestra.run.mira079:Running: "if test '!' -e /home/ubuntu/cephtest/archive/coredump ; then echo OK ; fi" 2014-07-01 17:22:40,833.833 INFO:teuthology.orchestra.run.mira043:Running: "if test '!' -e /home/ubuntu/cephtest/archive/coredump ; then echo OK ; fi" 2014-07-01 17:22:41,213.213 INFO:teuthology.orchestra.run.mira028:Running: "if test '!' -e /home/ubuntu/cephtest/archive/coredump ; then echo OK ; fi" 2014-07-01 17:22:41,595.595 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2014-07-01 17:22:41,595.595 INFO:teuthology.task.internal:Removing archive directory... 2014-07-01 17:22:41,595.595 INFO:teuthology.orchestra.run.mira028:Running: 'rm -rf -- /home/ubuntu/cephtest/archive' 2014-07-01 17:22:42,181.181 INFO:teuthology.orchestra.run.mira043:Running: 'rm -rf -- /home/ubuntu/cephtest/archive' 2014-07-01 17:22:42,563.563 INFO:teuthology.orchestra.run.mira079:Running: 'rm -rf -- /home/ubuntu/cephtest/archive' 2014-07-01 17:22:42,953.953 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2014-07-01 17:22:42,953.953 INFO:teuthology.task.internal:Tidying up after the test... 2014-07-01 17:22:42,953.953 INFO:teuthology.orchestra.run.mira028:Running: 'rmdir -- /home/ubuntu/cephtest' 2014-07-01 17:22:43,334.334 INFO:teuthology.orchestra.run.mira043:Running: 'rmdir -- /home/ubuntu/cephtest' 2014-07-01 17:22:43,722.722 INFO:teuthology.orchestra.run.mira079:Running: 'rmdir -- /home/ubuntu/cephtest' 2014-07-01 17:22:44,063.063 INFO:teuthology.run:Summary data: {client.0-kernel-sha1: 8362a1290d075f376ba68521ffb3b42ecaaecfea, duration: 4417.4871780872345, flavor: basic, mon.a-kernel-sha1: 8362a1290d075f376ba68521ffb3b42ecaaecfea, osd.3-kernel-sha1: 8362a1290d075f376ba68521ffb3b42ecaaecfea, owner: loic@dachary.org, success: true} 2014-07-01 17:22:44,063.063 WARNING:teuthology.report:No job_id found; not reporting results 2014-07-01 17:22:44,063.063 INFO:teuthology.run:pass